diff --git a/deploy.sh b/deploy.sh index 795d271..3373e67 100755 --- a/deploy.sh +++ b/deploy.sh @@ -1,3 +1,17 @@ #!/usr/bin/bash +# Copyright 2026 jCloud Services GbR +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + scp dist/* jcloud@jcloud-services.ddns.net:/srv/data/jcloud/htdocs/simple/jeb-utils \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index a4e2cb3..a8b0397 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,4 +7,4 @@ name = "jeb-utils" version = "0.1.4" description = "Common utils for JEB client and server." dependencies = ["cryptography"] -license = "MIT" \ No newline at end of file +license = "Apache-2.0" \ No newline at end of file diff --git a/src/jeb_utils/auth_utils.py b/src/jeb_utils/auth_utils.py index 6141131..3bd1089 100644 --- a/src/jeb_utils/auth_utils.py +++ b/src/jeb_utils/auth_utils.py @@ -25,7 +25,7 @@ __all__ = [ def load_cert_file(path: str) -> x509.Certificate: ''' Loads a certificate from a file - + :param path: File path :type path: str :return: The certificate @@ -58,7 +58,7 @@ class Identity: @property def not_valid_after(self): return self.cert.not_valid_after_utc - + @property def signature_algorithm(self): return self.cert.signature_hash_algorithm @@ -81,8 +81,8 @@ class Verifier: ) except: return False - + if (identity.issuer != self.trusted_ca.subject) and verify_issuer: return False - + return True \ No newline at end of file diff --git a/src/jeb_utils/jeb_utils.py b/src/jeb_utils/jeb_utils.py index 690f8e1..392d488 100644 --- a/src/jeb_utils/jeb_utils.py +++ b/src/jeb_utils/jeb_utils.py @@ -78,7 +78,7 @@ log_end_offsets = {} def format_topic_partitions(topic_name: str, partitions: set) -> list: ''' Formats the topic partitions. - + :param topic_name: The name of the topic, e. g. ``test-topic`` :type topic_name: str :param partitions: The partitions, e. g. ``{1, 2, 4}`` @@ -92,7 +92,7 @@ def format_topic_partitions(topic_name: str, partitions: set) -> list: def mktopic(topic_name: str): ''' Creates a topic. - + :param topic_name: The name of the topic :type topic_name: str ''' @@ -107,7 +107,7 @@ def mktopic(topic_name: str): def rmtopic(topic_name: str): ''' Removes a topic. - + :param topic_name: The topic name :type topic_name: str @@ -130,7 +130,7 @@ def rmtopic(topic_name: str): def mktopicpart(topic_name: str, partition_number: int): ''' Creates a partition of a topic. - + :param topic_name: The topic name :type topic_name: str :param partition_number: The partition number @@ -153,7 +153,7 @@ def mktopicpart(topic_name: str, partition_number: int): def rmtopicpart(topic_name: str, partition_number: int): ''' Removes a partition of a topic. - + :param topic_name: The topic name :type topic_name: str :param partition_number: The partition number @@ -194,7 +194,7 @@ def _create_base_segment_files(topics: dict): def get_segment_base_timestamp(topic: str, segment: int) -> int: ''' Returns the base timestamp of a segment. - + :param topic: The topic :type topic: str :param segment: The segment base offset @@ -216,7 +216,7 @@ def get_segment_base_timestamp(topic: str, segment: int) -> int: timestamp = None else: timestamp = int.from_bytes(timestamp_bytes) - + return timestamp def get_log_end_offsets(topics: dict): @@ -247,7 +247,7 @@ def get_log_end_offsets(topics: dict): def init(config: dict = {'segment_max_size': 1024 * 256, 'data_dir': './data'}): ''' Initializes the module. Calling this function is required before using most of the functions of the module. - + :param config: The configuration :type config: dict ''' @@ -269,7 +269,7 @@ def get_next_lower_index_entry(offset: int): def validate_topic_partition(partition_number: str | int) -> bool: ''' Validates a topic partition number format. - + :param partition_number: The partition number :type partition_number: str | int @@ -283,16 +283,16 @@ def validate_topic_partition(partition_number: str | int) -> bool: elif not isinstance(partition_number, int): return False - + if not (0 < partition_number < 100): return False - + return True def validate_topic_name(topic_name: str) -> bool: ''' Validates a topic name. - + :param topic_name: The topic name :type topic_name: str @@ -301,16 +301,16 @@ def validate_topic_name(topic_name: str) -> bool: ''' if not isinstance(topic_name, str): return False - + if not _TOPIC_NAME_REGEX.match(topic_name): return False - + if len(topic_name) == 0 or len(topic_name) > 255: return False - + if topic_name.startswith('.') or topic_name.endswith('.') or topic_name.startswith('..') or topic_name.endswith('..'): return False - + return True def validate_topic_format(topic: str): @@ -319,11 +319,11 @@ def validate_topic_format(topic: str): _, formatted_partition_number = topic.split(TOPIC_PARTITION_SEPARATOR) if len(formatted_partition_number) != 2 or not formatted_partition_number.isdigit(): raise ValueError('invalid partition number format') - + def check_topic_exists(topic: str): ''' Checks whether a topic exists. - + :param topic: The topic name :type topic: str @@ -368,7 +368,7 @@ def _unpack_record_headers(buffer: io.BytesIO, number: int = 256): key = buffer.read(key_length) if len(key) < key_length: break - + value_length_bytes = buffer.read(4) if len(value_length_bytes) < 4: break @@ -384,7 +384,7 @@ def _unpack_record_headers(buffer: io.BytesIO, number: int = 256): def unpack_record_headers(packed: bytes) -> dict: ''' Unpacks the record headers from bytes. - + :param packed: The packed headers :type packed: bytes @@ -397,7 +397,7 @@ def unpack_record_headers(packed: bytes) -> dict: def create_record(topic: str, timestamp: int, record_data: bytes, key: bytes = b'', compression_type: int = 0, headers: dict = {}): ''' Creates a record. - + :param topic: The topic name :type topic: str :param timestamp: The timestamp. @@ -416,7 +416,7 @@ def create_record(topic: str, timestamp: int, record_data: bytes, key: bytes = b topic_name, formatted_partition_number = topic.split(TOPIC_PARTITION_SEPARATOR) check_topic_exists(topic) - + topic_dir = f'{DATA_DIR}/topics/{topic_name}{TOPIC_PARTITION_SEPARATOR}{formatted_partition_number}/' utils.create_file_if_not_exists(topic_dir + '0' * 16 + '.log') segments = sorted([int(f.split('.')[0], 16) for f in os.listdir(topic_dir) if f.endswith('.log') and utils.is_number(f.split('.')[0], 16) and len(f.split('.')[0]) == 16]) @@ -447,7 +447,7 @@ def create_record(topic: str, timestamp: int, record_data: bytes, key: bytes = b record += key record += len(value).to_bytes(4) record += value - + record = len(record).to_bytes(8) + record # Prepend Record Size @@ -460,7 +460,7 @@ def create_record(topic: str, timestamp: int, record_data: bytes, key: bytes = b utils.create_file_if_not_exists(topic_dir + f'{last_segment:016x}.index') utils.create_file_if_not_exists(topic_dir + f'{last_segment:016x}.timeindex') - + if log_end_offsets[topic] % 1024 == 0: index = log_end_offsets[topic].to_bytes(8) + os.path.getsize(topic_dir + f'{last_segment:016x}.log').to_bytes(4) timeindex = timestamp.to_bytes(8) + log_end_offsets[topic].to_bytes(8) @@ -476,7 +476,7 @@ def create_record(topic: str, timestamp: int, record_data: bytes, key: bytes = b async def fetch_records(topic: str, start_type: int, start: int, max_bytes: int, send_block_function: FunctionType | MethodType): ''' Fetches records from the topic and sends it using ``send_block_function`` - + :param topic: The topic name :type topic: str :param start_type: The start type @@ -509,7 +509,7 @@ async def fetch_records(topic: str, start_type: int, start: int, max_bytes: int, if len(timestamp_bytes) < 8: break timestamp = int.from_bytes(timestamp_bytes) - + offset_bytes = f.read(8) if len(offset_bytes) < 8: break @@ -603,7 +603,7 @@ async def fetch_records(topic: str, start_type: int, start: int, max_bytes: int, - + await send_block_function(b'\x00') # EOF class Topic: @@ -612,11 +612,11 @@ class Topic: def __repr__(self): return f'Topic({self.topic_name})' - + @property def segments(self): return [Segment(self.topic_name, bo) for bo in sorted([int(s.split('.')[0], 16) for s in os.listdir(f'{DATA_DIR}/topics/{self.topic_name}') if s.endswith('.log') and utils.is_number(s.split('.')[0], 16) and len(s.split('.')[0]) == 16])] - + class Segment: def __init__(self, topic: str, base_offset: int): self.topic = topic @@ -633,7 +633,7 @@ class Segment: timestamp = None else: timestamp = int.from_bytes(timestamp_bytes) - + return timestamp class FileCorruptWarning(UserWarning): ... diff --git a/src/jeb_utils/jebp_utils.py b/src/jeb_utils/jebp_utils.py index fc7b34f..3d1faee 100644 --- a/src/jeb_utils/jebp_utils.py +++ b/src/jeb_utils/jebp_utils.py @@ -33,7 +33,7 @@ class MessageFormatError(Exception): ... async def sendmsg(m: bytes, writer: asyncio.StreamWriter, aesgcm: AESGCM = None, aesnonce: bytes = None, send_headers: bool = True, _content_length: int = None, start_byte: bytes = _MSG_START_BYTE): ''' Sends a message. - + :param m: The message :type m: bytes :param writer: The writer @@ -66,7 +66,7 @@ async def sendmsg(m: bytes, writer: asyncio.StreamWriter, aesgcm: AESGCM = None, async def readmsg(reader: asyncio.StreamReader, aesgcm: AESGCM = None, aesnonce: bytes = None, start_byte: bytes = _MSG_START_BYTE) -> bytes: ''' Receives a message. - + :param reader: The reader :type reader: asyncio.StreamReader :param aesgcm: AESGCM @@ -93,7 +93,7 @@ async def readmsg(reader: asyncio.StreamReader, aesgcm: AESGCM = None, aesnonce: def pack_fields(*fields: bytes) -> bytes: ''' Packs the fields into a compact bytestring. - + :param fields: The fields :type fields: bytes @@ -110,7 +110,7 @@ def pack_fields(*fields: bytes) -> bytes: def unpack_fields(raw: bytes) -> Sequence[bytes]: ''' Unpacks the field from a bytestring. - + :param raw: The packed fields :type raw: bytes diff --git a/src/jeb_utils/utils.py b/src/jeb_utils/utils.py index 9d1dae5..44e210d 100644 --- a/src/jeb_utils/utils.py +++ b/src/jeb_utils/utils.py @@ -28,7 +28,7 @@ __all__ = [ def is_number(string: str, base: int = 10) -> bool: ''' Checks whether the string is a number. - + :param string: The string :type string: str :param base: The base @@ -48,7 +48,7 @@ def is_number(string: str, base: int = 10) -> bool: def int_to_bytes(n: int, signed: bool = False) -> bytes: ''' Converts the integer into bytes (dynamic length). - + :param n: The integer :type n: int :param signed: Whether the integer is signed or not. @@ -76,7 +76,7 @@ def _bits_to_byte(bits: Tuple[Literal[0, 1], Literal[0, 1], Literal[0, 1], Liter def bits_to_byte(*bits: Literal[0, 1]) -> bytes: ''' Converts the bits into a single byte. - + :param bits: The bits :type bits: int @@ -94,7 +94,7 @@ def bits_to_byte(*bits: Literal[0, 1]) -> bytes: def byte_to_bits(b: bytes) -> Tuple[Literal[0, 1], Literal[0, 1], Literal[0, 1], Literal[0, 1], Literal[0, 1], Literal[0, 1], Literal[0, 1], Literal[0, 1]]: ''' Converts the byte to a tuple of bits. - + :param b: The byte :type b: bytes @@ -111,7 +111,7 @@ def byte_to_bits(b: bytes) -> Tuple[Literal[0, 1], Literal[0, 1], Literal[0, 1], def get_next_lower_integer_multiple(value: int, multiple: int) -> int: ''' Returns the next lower integer multiple. - + :param value: The value :type value: int :param multiple: The factor @@ -125,7 +125,7 @@ def get_next_lower_integer_multiple(value: int, multiple: int) -> int: def find_nearest_lower_number(number_list: Sequence, target: int | float) -> int | float: ''' Finds the nearest lower number to the target from the list. - + :param number_list: The list :type number_list: Sequence :param target: The target @@ -144,7 +144,7 @@ def find_nearest_lower_number(number_list: Sequence, target: int | float) -> int def create_file_if_not_exists(path: str, content: bytes = b''): ''' Creates a file if it does not exist. - + :param path: The file path :type path: str :param content: Optional, the content of the file if it is newly created.