From 7c4880a78e5518a92b0a2d4c20d0e64ba698e8b2 Mon Sep 17 00:00:00 2001 From: Jakob Scheid Date: Mon, 9 Mar 2026 22:17:33 +0100 Subject: [PATCH] First release of the new jeb core library --- NOTICE | 30 ++ README.md | 36 +++ deploy.sh | 18 ++ licenses/cffi.txt | 22 ++ licenses/config-parser.txt | 215 +++++++++++++ licenses/cryptography.APACHE.txt | 202 ++++++++++++ licenses/cryptography.BSD.txt | 27 ++ licenses/jeb-utils.txt | 215 +++++++++++++ licenses/pycparser.txt | 27 ++ pyproject.toml | 10 + src/jeb_core/__init__.py | 24 ++ src/jeb_core/_core.py | 517 +++++++++++++++++++++++++++++++ 12 files changed, 1343 insertions(+) create mode 100755 deploy.sh create mode 100644 licenses/cffi.txt create mode 100644 licenses/config-parser.txt create mode 100644 licenses/cryptography.APACHE.txt create mode 100644 licenses/cryptography.BSD.txt create mode 100644 licenses/jeb-utils.txt create mode 100644 licenses/pycparser.txt create mode 100644 pyproject.toml create mode 100644 src/jeb_core/__init__.py create mode 100644 src/jeb_core/_core.py diff --git a/NOTICE b/NOTICE index e69de29..5ed03cd 100644 --- a/NOTICE +++ b/NOTICE @@ -0,0 +1,30 @@ +jCloud Third-Party Notices +=========================== + +This software uses the following third-party libraries. Full license texts are available in the 'licenses/' directory. + +1. config-parser 1.2.1 + Author: jCloud Services GbR + License: Apache License 2.0 + See licenses/config-parser.txt for full license text. + +2. jeb-utils 0.1.4 + Author: jCloud Services GbR + License: Apache License 2.0 + See licenses/jeb-utils.txt for full license text. + +3. cffi 2.0.0 + Author: Armin Rigo, Maciej Fijalkowski + License: MIT License + See licenses/cffi.txt for full license text. + +4. cryptography 46.0.5 + Author: Python Cryptographic Authority + License: Apache License 2.0 or BSD 3-Clause License + URL: https://cryptography.io/ + See licenses/cryptography.APACHE.txt or licenses/cryptography.BSD.txt for full license text. + +5. pycparser 3.0 + Author: Eli Bendersky + License: BSD 3-Clause License + See licenses/pycparser.txt for full license text. \ No newline at end of file diff --git a/README.md b/README.md index e69de29..e4359e5 100644 --- a/README.md +++ b/README.md @@ -0,0 +1,36 @@ +# jeb-core + +The core for jeb + +## Installation +You can install the library using `pip`: + +```bash +pip install jeb-core --index-url https://repo.jcloud-services.ddns.net/simple/ --extra-index-url https://pypi.org/simple +``` + +## Usage +### `JEBCore` +The main class. It has following methods: +- `get_segment_base_timestamp`: Returns the base timestamp of a segment. +- `get_topics`: Returns all topics. +- `mktopic`: Creates a topic. +- `rmtopic`: Removes a topic. +- `mktopicpart`: Creates a partition of a topic. +- `rmtopicpart`: Removes a partition of a topic. +- `check_topic_exists`: Checks whether a topic exists (raises an exception if not). +- `create_record`: Creates a record. +- `fetch_records`: Fetches records from the topic and yields them in chunks. + +### `Topic` +A topic. + +### `Segment`: +A segment. + +For more details, see the function and method docstrings. + +## Changelog + +### Version 0.1.0 +- First release of the new jeb core library \ No newline at end of file diff --git a/deploy.sh b/deploy.sh new file mode 100755 index 0000000..4b2893b --- /dev/null +++ b/deploy.sh @@ -0,0 +1,18 @@ +#!/usr/bin/bash + +# Copyright 2026 jCloud Services GbR +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +python3 -m build +scp dist/* jcloud@jcloud-services.ddns.net:/srv/data/wwwstatic/repo/simple/jeb-core/ \ No newline at end of file diff --git a/licenses/cffi.txt b/licenses/cffi.txt new file mode 100644 index 0000000..51ac1b9 --- /dev/null +++ b/licenses/cffi.txt @@ -0,0 +1,22 @@ + +Except when otherwise stated (look for LICENSE files in directories or +information at the beginning of each file) all software and +documentation is licensed as follows: + + MIT No Attribution + + Permission is hereby granted, free of charge, to any person + obtaining a copy of this software and associated documentation + files (the "Software"), to deal in the Software without + restriction, including without limitation the rights to use, + copy, modify, merge, publish, distribute, sublicense, and/or + sell copies of the Software, and to permit persons to whom the + Software is furnished to do so. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. diff --git a/licenses/config-parser.txt b/licenses/config-parser.txt new file mode 100644 index 0000000..cc1733c --- /dev/null +++ b/licenses/config-parser.txt @@ -0,0 +1,215 @@ +Copyright 2026 jCloud Services GbR + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/licenses/cryptography.APACHE.txt b/licenses/cryptography.APACHE.txt new file mode 100644 index 0000000..e25e752 --- /dev/null +++ b/licenses/cryptography.APACHE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + https://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/licenses/cryptography.BSD.txt b/licenses/cryptography.BSD.txt new file mode 100644 index 0000000..86dddf1 --- /dev/null +++ b/licenses/cryptography.BSD.txt @@ -0,0 +1,27 @@ +Copyright (c) Individual contributors. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + 3. Neither the name of PyCA Cryptography nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file diff --git a/licenses/jeb-utils.txt b/licenses/jeb-utils.txt new file mode 100644 index 0000000..cc1733c --- /dev/null +++ b/licenses/jeb-utils.txt @@ -0,0 +1,215 @@ +Copyright 2026 jCloud Services GbR + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/licenses/pycparser.txt b/licenses/pycparser.txt new file mode 100644 index 0000000..a89b5fa --- /dev/null +++ b/licenses/pycparser.txt @@ -0,0 +1,27 @@ +pycparser -- A C parser in Python + +Copyright (c) 2008-2022, Eli Bendersky +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. +* Neither the name of the copyright holder nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE +GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT +OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..116f793 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,10 @@ +[build-system] +requires = ["setuptools", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "jeb-core" +version = "0.1.0" +description = "The core for jeb" +dependencies = ["config-parser", "jeb-utils"] +license = "Apache-2.0" \ No newline at end of file diff --git a/src/jeb_core/__init__.py b/src/jeb_core/__init__.py new file mode 100644 index 0000000..3429074 --- /dev/null +++ b/src/jeb_core/__init__.py @@ -0,0 +1,24 @@ +# Copyright 2026 jCloud Services GbR +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +''' +The core for jeb +''' + +from ._core import * +from ._core import __all__ as _core__all__ + +__all__ = [ + *_core__all__ +] \ No newline at end of file diff --git a/src/jeb_core/_core.py b/src/jeb_core/_core.py new file mode 100644 index 0000000..96e19c8 --- /dev/null +++ b/src/jeb_core/_core.py @@ -0,0 +1,517 @@ +# Copyright 2026 jCloud Services GbR +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from jeb_utils import exceptions, jeb_utils, utils +import os +import dbm +import config_parser +import shutil +import time +import binascii +import warnings + +__all__ = [ + 'JEBCore', + 'Segment', + 'Topic', +] + +def _create_file_if_not_exists(path: str, content: bytes = b''): + ''' + Creates a file if it does not exist. + + :param path: The file path + :type path: str + :param content: Optional, the content of the file if it is newly created. + :type content: bytes + ''' + if not os.path.exists(path): + with open(path, 'wb') as f: + f.write(content) + f.close() + +class JEBCore: + def __init__(self, data_dir: str, *, segment_max_size: int = 2 ** 32, fetch_records_chunk_size: int = 1024 ** 2) -> None: + ''' + The core for jeb. + + :param data_dir: The data directory + :type data_dir: str + :param segment_max_size: The maximum size for segments + :type segment_max_size: int + :param fetch_records_chunk_size: The size of the chunks while fetching the records + :type fetch_records_chunk_size: int + ''' + self.data_dir = data_dir + self.segment_max_size = segment_max_size + self.fetch_records_chunk_size = fetch_records_chunk_size + self._log_end_offsets = {} + self._init() + + def _init(self) -> None: + self._create_base_segment_files() + self._init_log_end_offsets() + + def _create_base_segment_files(self) -> None: + ''' + Creates the base segment files. + ''' + for topic, v in self.get_topics().items(): + partitions = jeb_utils.format_topic_partitions(topic, v['partitions']) + for p in partitions: + _create_file_if_not_exists(f'{self.data_dir}/topics/{p}/' + '0' * 16 + '.log') + _create_file_if_not_exists(f'{self.data_dir}/topics/{p}/' + '0' * 16 + '.index') + _create_file_if_not_exists(f'{self.data_dir}/topics/{p}/' + '0' * 16 + '.timeindex') + + def _init_log_end_offsets(self) -> None: + ''' + Initializes the log end offsets. + ''' + global _log_end_offsets + topics = [f'{t}{jeb_utils.TOPIC_PARTITION_SEPARATOR}{p:02d}' for t, v in self.get_topics().items() for p in v['partitions']] + topic_segments = {t: Topic(t, self).segments for t in topics} + for topic, segments in topic_segments.items(): + log_end_offset = 0 + for segment in segments: + with open(f'{self.data_dir}/topics/{topic}/{segment.base_offset:016x}.log', 'rb') as logfile: + logfile.seek(0, os.SEEK_END) + file_size = logfile.tell() + logfile.seek(0) + + while True: + record_size_bytes = logfile.read(8) + if not record_size_bytes or len(record_size_bytes) < 8: + break + record_size = int.from_bytes(record_size_bytes) + if logfile.tell() + record_size > file_size: + break + logfile.seek(record_size, os.SEEK_CUR) + log_end_offset += 1 + logfile.close() + self._log_end_offsets[topic] = log_end_offset + + def get_segment_base_timestamp(self, topic: str, segment: int) -> int: + ''' + Returns the base timestamp of a segment. + + :param topic: The topic + :type topic: str + :param segment: The segment base offset + :type segment: int + + :return: The base timestamp (milliseconds since 1970) + :rtype: int + ''' + with open(f'{self.data_dir}/topics/{topic}/{segment:016x}.log', 'rb') as logfile: + # go to the topic length + logfile.seek(20) + + # skip topic + topic_length = int.from_bytes(logfile.read(2)) + logfile.seek(topic_length + 1, os.SEEK_CUR) + + timestamp_bytes = logfile.read(8) + if len(timestamp_bytes) < 8: + timestamp = None + else: + timestamp = int.from_bytes(timestamp_bytes) + + return timestamp + + def get_topics(self) -> dict: + ''' + Returns all topics. + + :return: The topics + :rtype: dict + ''' + with dbm.open(f'{self.data_dir}/conf/topics', 'c') as db: + topics = {k.decode(): config_parser.parse.json.parse_json(v.decode()) for k, v in db.items()} + db.close() + return topics + + def mktopic(self, topic_name: str) -> None: + ''' + Creates a topic. + + :param topic_name: The name of the topic + :type topic_name: str + ''' + with dbm.open(f'{self.data_dir}/conf/topics', 'c') as db: + if topic_name.encode() in db.keys(): + raise exceptions.TopicExistsError(f'topic \'{topic_name}\' already exists') + db[topic_name] = config_parser.serialize.json.serialize({'partitions': [1]}) + db.close() + + os.mkdir(f'{self.data_dir}/topics/{topic_name}{jeb_utils.TOPIC_PARTITION_SEPARATOR}01/') + + def rmtopic(self, topic_name: str) -> None: + ''' + Removes a topic. + + :param topic_name: The topic name + :type topic_name: str + + :raises TopicNotFoundError: If the topic does not exist. + ''' + with dbm.open(f'{self.data_dir}/conf/topics', 'c') as db: + if topic_name.encode() not in db.keys(): + raise exceptions.TopicNotFoundError(f'topic \'{topic_name}\' does not exist') + partitions = config_parser.parse.json.parse_json(db[topic_name.encode()].decode())['partitions'] + del db[topic_name] + db.close() + + partition_dirs = jeb_utils.format_topic_partitions(topic_name, partitions) + for pd in partition_dirs: + try: + shutil.rmtree(f'{self.data_dir}/topics/{pd}/') + except: + pass + + def mktopicpart(self, topic_name: str, partition_number: int) -> None: + ''' + Creates a partition of a topic. + + :param topic_name: The topic name + :type topic_name: str + :param partition_number: The partition number + :type partition_number: int + ''' + if partition_number < 1 or partition_number > 99: + raise ValueError('partition number must be between 1 and 99') + formatted_partition_number = f'{(int(partition_number)):02d}' + with dbm.open(f'{self.data_dir}/conf/topics', 'c') as db: + if topic_name.encode() not in db.keys(): + raise exceptions.TopicNotFoundError(f'topic \'{topic_name}\' does not exist') + partitions = config_parser.parse.json.parse_json(db[topic_name.encode()].decode())['partitions'] + if partition_number in partitions: + raise exceptions.TopicPartitionExistsError(f'partition \'{formatted_partition_number}\' already exists for topic \'{topic_name}\'') + db[topic_name] = config_parser.serialize.json.serialize({'partitions': sorted(list(set(partitions) | {int(formatted_partition_number)}))}) + db.close() + + os.mkdir(f'{self.data_dir}/topics/{topic_name}{jeb_utils.TOPIC_PARTITION_SEPARATOR}{formatted_partition_number}/') + + def rmtopicpart(self, topic_name: str, partition_number: int) -> None: + ''' + Removes a partition of a topic. + + :param topic_name: The topic name + :type topic_name: str + :param partition_number: The partition number + :type partition_number: int + ''' + formatted_partition_number = f'{(int(partition_number)):02d}' + with dbm.open(f'{self.data_dir}/conf/topics', 'c') as db: + if topic_name.encode() not in db.keys(): + raise exceptions.TopicNotFoundError(f'topic \'{topic_name}\' does not exist') + partitions = config_parser.parse.json.parse_json(db[topic_name.encode()].decode())['partitions'] + if partition_number not in partitions: + raise exceptions.TopicPartitionNotFoundError(f'partition \'{formatted_partition_number}\' does not exist for topic \'{topic_name}\'') + db[topic_name] = config_parser.serialize.json.serialize({'partitions': sorted(list(set(partitions) - {int(formatted_partition_number)}))}) + db.close() + + shutil.rmtree(f'{self.data_dir}/topics/{topic_name}{jeb_utils.TOPIC_PARTITION_SEPARATOR}{formatted_partition_number}/') + + def check_topic_exists(self, topic: str) -> None: + ''' + Checks whether a topic exists. + + :param topic: The topic name + :type topic: str + + :raises TopicNotFoundError: If the topic does not exist + :raises TopicPartitionNotFoundError: If the partition does not exist + ''' + topics = self.get_topics() + topic_name, formatted_partition_number = topic.split(jeb_utils.TOPIC_PARTITION_SEPARATOR) + raw_partition_number = int(formatted_partition_number) + if topic_name not in topics: + raise exceptions.TopicNotFoundError(f'topic \'{topic_name}\' does not exist') + if raw_partition_number not in topics[topic_name]['partitions']: + raise exceptions.TopicPartitionNotFoundError(f'partition \'{formatted_partition_number}\' does not exist for topic \'{topic_name}\'') + + def create_record(self, topic: str, timestamp: int, record_data: bytes, key: bytes = b'', compression_type: int = 0, headers: dict = {}) -> None: + ''' + Creates a record. + + :param topic: The topic name + :type topic: str + :param timestamp: The timestamp. + :type timestamp: int + :param record_data: The payload + :type record_data: bytes + :param key: The key + :type key: bytes + :param compression_type: The compression type + :type compression_type: int + :param headers: The header dictionary + :type headers: dict + ''' + topic_name, formatted_partition_number = topic.split(jeb_utils.TOPIC_PARTITION_SEPARATOR) + + if not jeb_utils.validate_topic_name(topic_name): + raise exceptions.FormatError(f'invalid topic name: \'{topic}\'') + + self.check_topic_exists(topic) + + topic_dir = f'{self.data_dir}/topics/{topic_name}{jeb_utils.TOPIC_PARTITION_SEPARATOR}{formatted_partition_number}/' + utils.create_file_if_not_exists(topic_dir + '0' * 16 + '.log') + segments = sorted([int(f.split('.')[0], 16) for f in os.listdir(topic_dir) if f.endswith('.log') and utils.is_number(f.split('.')[0], 16) and len(f.split('.')[0]) == 16]) + last_segment = max(segments) + + if timestamp == 0: + timestamp = int(time.time() * 1000) + timestamp_type = 1 + else: + timestamp_type = 0 + + # Assemble record + + record = b'' + record += self._log_end_offsets[topic].to_bytes(8) # Offset + record += binascii.crc32(record_data).to_bytes(4) # CRC + record += (len(topic)).to_bytes(2) # Topic name length + record += topic.encode() # Topic + record += jeb_utils.AttributesByte(compression_type, timestamp_type).to_byte() # Attributes + record += timestamp.to_bytes(8) # Timestamp + record += (len(key)).to_bytes(4) # Key Length + record += key # Key + record += (len(record_data)).to_bytes(4) # Content Length + record += record_data # Content + record += (len(headers)).to_bytes(1) + for key, value in headers.items(): + record += len(key).to_bytes(4) + record += key + record += len(value).to_bytes(4) + record += value + + + record = len(record).to_bytes(8) + record # Prepend Record Size + + if os.path.getsize(topic_dir + f'{last_segment:016x}.log') + len(record) > self.segment_max_size: + last_segment = self._log_end_offsets[topic] + + with open(topic_dir + f'{last_segment:016x}.log', 'ab') as logfile: + logfile.write(record) + logfile.close() + + utils.create_file_if_not_exists(topic_dir + f'{last_segment:016x}.index') + utils.create_file_if_not_exists(topic_dir + f'{last_segment:016x}.timeindex') + + if self._log_end_offsets[topic] % 1024 == 0: + index = self._log_end_offsets[topic].to_bytes(8) + os.path.getsize(topic_dir + f'{last_segment:016x}.log').to_bytes(4) + timeindex = timestamp.to_bytes(8) + self._log_end_offsets[topic].to_bytes(8) + with open(topic_dir + f'{last_segment:016x}.index', 'ab') as idxfile: + idxfile.write(index) + idxfile.close() + with open(topic_dir + f'{last_segment:016x}.timeindex', 'ab') as timeidxfile: + timeidxfile.write(timeindex) + timeidxfile.close() + + self._log_end_offsets[topic] += 1 + + def fetch_records(self, topic: str, start_type: int, start: int, max_bytes: int): + ''' + Fetches records from the topic and yields them in chunks. + + :param topic: The topic name + :type topic: str + :param start_type: The start type + :type start_type: int + :param start: The start + :type start: int + :param max_bytes: The maximum of bytes + :type max_bytes: int + ''' + topic = Topic(topic, self) + topic_name, formatted_partition_number = topic.topic_name.split(jeb_utils.TOPIC_PARTITION_SEPARATOR) + if not jeb_utils.validate_topic_name(topic_name): + raise exceptions.FormatError(f'invalid topic name: \'{topic.topic_name}\'') + if start_type not in (jeb_utils.FETCH_START_TYPE_OFFSET, jeb_utils.FETCH_START_TYPE_TIMESTAMP): + raise ValueError('invalid start type') + self.check_topic_exists(topic.topic_name) + segments = [int(f.split('.')[0], 16) for f in os.listdir(f'{self.data_dir}/topics/{topic.topic_name}/') if utils.is_number(f.split('.')[0], 16) and len(f.split('.')[0]) == 16 and f.endswith('.log')] + + if start_type == jeb_utils.FETCH_START_TYPE_TIMESTAMP: + segment_base_timestamps = {s: s.base_timestamp for s in topic.segments} + s = {v: k for k, v in segment_base_timestamps.items()}[utils.find_nearest_lower_number(list(segment_base_timestamps.values()), start) or min(segment_base_timestamps.values())] + with open(f'{self.data_dir}/topics/{topic.topic_name}/{s.base_offset:016x}.timeindex', 'rb') as f: + timestamps = {} + f.seek(0) + while True: + timestamp_bytes = f.read(8) + if len(timestamp_bytes) < 8: + break + timestamp = int.from_bytes(timestamp_bytes) + + offset_bytes = f.read(8) + if len(offset_bytes) < 8: + break + offset = int.from_bytes(offset_bytes) + + timestamps[timestamp] = offset + f.close() + start = timestamps[utils.find_nearest_lower_number(timestamps.keys(), start) or min(timestamps.keys())] + + + if start < 0 or start >= self._log_end_offsets[topic.topic_name]: + # await send_block_function(b'\xb1\x31') + yield b'\xb1\x31' + return + idx_offset = jeb_utils.get_next_lower_index_entry(start) + + + start_seg = utils.find_nearest_lower_number(segments, start) + + + with open(f'{self.data_dir}/topics/{topic.topic_name}/' + f'{start_seg:016x}' + '.index', 'rb') as idxfile: + idxfile.seek(0, os.SEEK_END) + file_size = idxfile.tell() + idxfile.seek(idx_offset // 1024 * 12) + index_entry = idxfile.read(12) + idx_entry_pos = int.from_bytes(index_entry[8:]) + if int.from_bytes(index_entry[:8]) != idx_offset: + warnings.warn(f'Index entry offset mismatch: expected {idx_offset}, got {int.from_bytes(index_entry[:8])}', jeb_utils.FileCorruptWarning) + idxfile.close() + + with open(f'{self.data_dir}/topics/{topic.topic_name}/{start_seg:016x}.log', 'rb') as logfile: + logfile.seek(idx_entry_pos) + offset = idx_offset + record_position = 0 + while offset < start: + record_length_bytes = logfile.read(8) + if len(record_length_bytes) < 8: + break + record_length = int.from_bytes(record_length_bytes) + + offset_bytes = logfile.read(8) + if len(offset_bytes) < 8: + break + offset = int.from_bytes(offset_bytes) + + record_position = logfile.tell() - 16 + logfile.seek(record_length - 8, os.SEEK_CUR) + logfile.close() + + + # await send_block_function(b'\xa0\x30') + yield b'\xa0\x30' + + segments = sorted(segments) + + bytes_fetched = 0 + for i, segment in enumerate(segments[segments.index(start_seg):]): + records_sent = 0 + with open(f'{self.data_dir}/topics/{topic.topic_name}/' + f'{segment:016x}' + '.log', 'rb') as logfile: + if i == 0: + logfile.seek(record_position) + else: + logfile.seek(0) + while bytes_fetched <= max_bytes: + record_size_bytes = logfile.read(8) + if len(record_size_bytes) < 8: + break + record_size = int.from_bytes(record_size_bytes) + if bytes_fetched + 8 + record_size > max_bytes: + break + + to_send = record_size_bytes + + for _ in range(record_size // self.fetch_records_chunk_size): + to_send += logfile.read(self.fetch_records_chunk_size) + # await send_block_function(b'\x01' + to_send) + yield b'\x01' + to_send + to_send = b'' + to_send += logfile.read(record_size % self.fetch_records_chunk_size) + # await send_block_function(b'\x01' + to_send) + yield b'\x01' + to_send + + bytes_fetched += 8 + record_size + records_sent += 1 + + logfile.close() + + + + if bytes_fetched >= max_bytes: + break + + + + + + # await send_block_function(b'\x00') # EOF + yield b'\x00' # EOF + +class Segment: + def __init__(self, topic: str, base_offset: int, jeb_core: JEBCore) -> None: + ''' + Represents a segment of a topic. + + :param topic: The topic name + :type topic: str + :param base_offset: The base offset of the segment + :type base_offset: int + :param jeb_core: The ``JEBCore`` instance + :type jeb_core: JEBCore + ''' + self.topic = topic + self.base_offset = base_offset + self.jeb_core = jeb_core + self.base_timestamp = self.get_base_timestamp() + + def __repr__(self) -> str: + return f'Segment({self.topic}, {self.base_offset})' + + def get_base_timestamp(self) -> int: + ''' + Returns the base timestamp of the segment. + :return: The base timestamp (milliseconds since 1970) + :rtype: int + ''' + # with open(f'{self.jeb_core.data_dir}/topics/{self.topic}/{self.base_offset:016x}.timeindex', 'rb') as logfile: + # timestamp_bytes = logfile.read(8) + # if len(timestamp_bytes) < 8: + # timestamp = None + # else: + # timestamp = int.from_bytes(timestamp_bytes) + + # return timestamp + return self.jeb_core.get_segment_base_timestamp(self.topic, self.base_offset) + +class Topic: + def __init__(self, topic_name: str, jeb_core: JEBCore) -> None: + ''' + Represents a topic. + + :param topic_name: The name of the topic + :type topic_name: str + :param jeb_core: The ``JEBCore`` instance + :type jeb_core: JEBCore + ''' + self.topic_name = topic_name + self.jeb_core = jeb_core + + def __repr__(self) -> str: + return f'Topic({self.topic_name})' + + @property + def segments(self) -> list: + ''' + Returns the segments of the topic. + + :return: The segments + :rtype: list + ''' + return [Segment(self.topic_name, bo, self.jeb_core) for bo in sorted([int(s.split('.')[0], 16) for s in os.listdir(f'{self.jeb_core.data_dir}/topics/{self.topic_name}') if s.endswith('.log') and utils.is_number(s.split('.')[0], 16) and len(s.split('.')[0]) == 16])] \ No newline at end of file