Merge pull request #11 from MatrixEditor/dev/2.2.0

[DEV] 2.2.0 - C API Breaking changes

.github/workflows/python-test.yml

@@ -13,17 +13,17 @@ jobs:
 
     steps:
       - name: Checkout source
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Setup python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
           architecture: x64
 
       - name: Install
         run: |
-          pip install -e .
+          pip install -ve ./src/ccaterpillar
           pip install -r requirements.txt
           pip install -r test/requirements.txt
 

CMakeLists.txt

@@ -8,33 +8,51 @@ find_package(
 
 add_compile_definitions(_CPMODULE)
 
+# TODO: document this change
+set(CP_C_CONFIGURED DEFINED ENV{CP_ENABLE_NATIVE} OR DEFINED CP_ENABLE_NATIVE)
 
-if (DEFINED ENV{CP_ENABLE_NATIVE})
+if (CP_C_CONFIGURED)
 python_add_library(
   _C
   MODULE
 
-  src/arch.c
-  src/atomobj.c
-  src/context.c
-  src/field.c
-  src/option.c
-  src/module.c
-  src/context.c
-  src/state.c
-  src/struct.c
-
-  src/parsing_typeof.c
-  src/parsing_sizeof.c
-  src/parsing_pack.c
-  src/parsing_unpack.c
-
-  src/atomimpl/intatomobj.c
-  src/atomimpl/floatatomobj.c
-  src/atomimpl/boolatomobj.c
-  src/atomimpl/charatomobj.c
-  src/atomimpl/padatomobj.c
-  src/atomimpl/stringatomobj.c
+  src/ccaterpillar/arch.c
+  src/ccaterpillar/atomobj.c
+  src/ccaterpillar/context.c
+  src/ccaterpillar/option.c
+  src/ccaterpillar/module.c
+  src/ccaterpillar/context.c
+  src/ccaterpillar/state.c
+  src/ccaterpillar/struct.c
+  src/ccaterpillar/layer.c
+  src/ccaterpillar/default.c
+
+  src/ccaterpillar/parsing_typeof.c
+  src/ccaterpillar/parsing_sizeof.c
+  src/ccaterpillar/parsing_pack.c
+  src/ccaterpillar/parsing_unpack.c
+
+  src/ccaterpillar/atomimpl/int.c
+  src/ccaterpillar/atomimpl/float.c
+  src/ccaterpillar/atomimpl/bool.c
+  src/ccaterpillar/atomimpl/char.c
+  src/ccaterpillar/atomimpl/pad.c
+  src/ccaterpillar/atomimpl/string.c
+  src/ccaterpillar/atomimpl/const.c
+  src/ccaterpillar/atomimpl/bytes.c
+  src/ccaterpillar/atomimpl/pstring.c
+  src/ccaterpillar/atomimpl/enum.c
+  src/ccaterpillar/atomimpl/varint.c
+  src/ccaterpillar/atomimpl/computed.c
+  src/ccaterpillar/atomimpl/lazy.c
+  src/ccaterpillar/atomimpl/cstring.c
+
+  src/ccaterpillar/atomimpl/builtins/builtin.c
+  src/ccaterpillar/atomimpl/builtins/repeated.c
+  src/ccaterpillar/atomimpl/builtins/condition.c
+  src/ccaterpillar/atomimpl/builtins/switch.c
+  src/ccaterpillar/atomimpl/builtins/atoffset.c
+  src/ccaterpillar/atomimpl/builtins/primitive.c
 
   WITH_SOABI
 )
@@ -44,10 +62,18 @@ target_include_directories(_C PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src/caterpilla
 message(STATUS "CMake ${CMAKE_VERSION}")
 message(STATUS "Python ${Python_VERSION_MAJOR}.${Python_VERSION_MINOR}")
 
+set (CP_GENAPI_SCRIPT ${CMAKE_CURRENT_SOURCE_DIR}/src/code_gen/genapi.py)
+set (CP_CAPI_HEADER ${CMAKE_CURRENT_SOURCE_DIR}/src/caterpillar/include/caterpillar/caterpillarapi.h.in)
+set (CP_CAPI_CSRC ${CMAKE_CURRENT_SOURCE_DIR}/src/ccaterpillar/caterpillarapi.c.in)
+set (CP_CAPI_HEADER_OUT ${CMAKE_CURRENT_SOURCE_DIR}/src/caterpillar/include/caterpillar/caterpillarapi.h)
+set (CP_CAPI_CSRC_OUT ${CMAKE_CURRENT_SOURCE_DIR}/src/ccaterpillar/caterpillarapi.c)
+
 add_custom_target(
  genapi ALL
- COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/src/code_gen/genapi.py ${CMAKE_CURRENT_SOURCE_DIR}/src/caterpillar/include/caterpillar/caterpillarapi.h.in ${CMAKE_CURRENT_SOURCE_DIR}/src/caterpillarapi.c.in
- BYPRODUCTS ${CMAKE_CURRENT_SOURCE_DIR}/src/caterpillar/include/caterpillar/caterpillarapi.h ${CMAKE_CURRENT_SOURCE_DIR}/src/caterpillarapi.c
+ # execute python ./src/code_gen/genapi.py ./src/caterpillar/include/caterpillar/caterpillarapi.h.in ./src/ccaterpillar/caterpillarapi.c.in
+ # in the root directory
+ COMMAND ${PYTHON_EXECUTABLE} ${CP_GENAPI_SCRIPT} ${CP_CAPI_HEADER} ${CP_CAPI_CSRC}
+ BYPRODUCTS ${CP_CAPI_HEADER_OUT} ${CP_CAPI_CSRC_OUT}
  COMMENT "Generating Public Caterpillar API"
 )
 

README.md

@@ -28,10 +28,10 @@ options will be added in the future. Documentation is [here >](https://matrixedi
 * it helps you to create cleaner and more compact code.
 * You can even extend Caterpillar and write your parsing logic in C or C++!!
 
-## What does it look like?
+## Give me some code!
 
 ```python
-from caterpillar._Py import *
+from caterpillar.py import *
 
 @struct(order=LittleEndian)
 class Format:
@@ -61,24 +61,19 @@ on its powerful features, explore the official [documentation](https://matrixedi
 > As of Caterpillar v2.1.2 it is possible to install the library without the need of
 > compiling the C extension.
 
-Simply use pip to install the package with all tools, all extra packages and the C extension:
+### Python-only installation
+
 ```bash
-pip install "caterpillar[all]@git+https://github.com/MatrixEditor/caterpillar.git"
+pip install "caterpillar[all]@git+https://github.com/MatrixEditor/caterpillar"
 ```
 
-If you want to use the native C extension you can specify the following environment variables:
-
-* `CP_ENABLE_NATIVE` - Enables installation of the native C extension
-* `CP_ENABLE_TOOLS` - Enables installation of extra tools (proposed)
+### C-extension installation
 
-For instance, the following command will install the `caterpillar` package with
-a native interface and with extra tools:
 ```bash
-CP_ENABLE_NATIVE=1 \
-CP_ENABLE_TOOLS=1 \
-pip install git+https://github.com/MatrixEditor/caterpillar
+pip install "caterpillar[all]@git+https://github.com/MatrixEditor/caterpillar/#subdirectory=src/ccaterpillar"
 ```
 
+
 ## Starting Point
 
 Please visit the [Documentation](https://matrixeditor.github.io/caterpillar/), it contains a complete tutorial on how to use this library.

docs/requirements.txt

@@ -1,4 +1,5 @@
 sphinx
 pydata-sphinx-theme
 sphinx-design
-breathe
+breathe
+sphinx-copybutton

docs/sphinx/source/conf.py

@@ -27,7 +27,8 @@
     "sphinx.ext.viewcode",
     "sphinx_design",
     "breathe",
-    "c_annotations"
+    "c_annotations",
+    "sphinx_copybutton"
 ]
 
 templates_path = ["_templates"]
@@ -61,7 +62,7 @@
 # If true, `todo` and `todoList` produce output, else they produce nothing.
 todo_include_todos = False
 
-refcount_file = 'extensions/refcounts.dat'
+refcount_file = '../../../src/capi.dat'
 
 # -- Options for HTML output -------------------------------------------------
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output

docs/sphinx/source/development/index.rst

@@ -10,5 +10,6 @@ Development
     :numbered:
     :maxdepth: 2
 
+    roadmap.rst
     contribution.rst
     changelog.rst

docs/sphinx/source/development/roadmap.rst

@@ -0,0 +1,111 @@
+.. _dev-roadmap:
+
+********
+Roadmap
+********
+
+.. |check_| raw:: html
+
+    <input checked=""  disabled="" type="checkbox">
+
+.. |uncheck_| raw:: html
+
+    <input disabled="" type="checkbox">
+
+.. role:: text-danger
+
+.. role:: text-warning
+
+Python API
+----------
+
+- |check_| Implementation of parsing process (unpack, pack)
+- |check_| Struct class (:class:`Struct`) with wrapper function (:code:`@struct`)
+- |uncheck_| Python docs and examples
+- |uncheck_| Python tests
+
+C API
+-----
+
+- |check_| Implementation of parsing process (unpack, pack)
+- |uncheck_| Struct class (:c:type:`CpStructObject`)
+- |uncheck_| Struct wrapper function
+- |uncheck_| Python docs
+
+Struct Objects:
+^^^^^^^^^^^^^^^
+
+- |check_| Struct (C type: :c:type:`CpStructObject`, Py type: :class:`Struct`)
+    [:text-danger:`missing docs`],
+    [:text-warning:`missing perftest`]
+
+- |uncheck_| Bitfield
+
+Atom Objects:
+^^^^^^^^^^^^^
+
+- |check_| Integer (uint8-128 and int8-128) (C type: :c:type:`CpIntAtomObject`, Py type: :class:`int_t`)
+    [:text-danger:`missing docs`]
+
+- |check_| Float, Double (C type: :c:type:`CpFloatAtomObject`, Py type: :class:`float_t`)
+    [:text-danger:`missing docs`]
+
+- |check_| Boolean (C type: :c:type:`CpBoolAtomObject`, Py type: :class:`bool_t`)
+    Global instance: :code:`boolean`,
+    [:text-danger:`missing docs`]
+
+- |check_| Char (C type: :c:type:`CpCharAtomObject`, Py type: :class:`char_t`)
+    Global instance: :code:`char`,
+    [:text-danger:`missing docs`]
+
+- |check_| Padding (C type: :c:type:`CpPaddingAtomObject`, Py type: :class:`padding_t`)
+    Global instance: :code:`padding`,
+    [:text-danger:`missing docs`]
+
+- |check_| String (C type: :c:type:`CpStringAtomObject`, Py type: :class:`string`)
+    [:text-danger:`missing docs`],
+    [:text-warning:`missing perftest`]
+
+- |check_| Const (C type: :c:type:`CpConstAtomObject`, Py type: :class:`const_t`)
+    [:text-danger:`missing docs`],
+    [:text-warning:`missing perftest`]
+
+- |uncheck_| CString (C type: :c:type:`CpCStringAtomObject`, Py type: :class:`cstring`)
+    [:text-danger:`missing docs`],
+    [:text-warning:`missing perftest`]
+
+- |check_| Bytes (C type: :c:type:`CpBytesAtomObject`, Py type: :class:`octetstring`)
+    [:text-danger:`missing docs`],
+    [:text-warning:`missing perftest`]
+
+- |check_| Enum (C type: :c:type:`CpEnumAtomObject`, Py type: :class:`enumeration`)
+    [:text-danger:`missing docs`],
+    [:text-warning:`missing perftest`]
+
+- |check_| Computed (C type: :c:type:`CpComputedAtomObject`, Py type: :class:`computed`)
+    [:text-danger:`missing docs`],
+    [:text-warning:`missing perftest`]
+
+- |check_| PString (C type: :c:type:`CpPStringAtomObject`, Py type: :class:`pstring`)
+    [:text-danger:`missing docs`],
+    [:text-warning:`missing perftest`]
+
+- |uncheck_| Prefixed
+    .. seealso:: *link issue here*
+
+- |check_| Lazy
+    [:text-danger:`missing docs`],
+    [:text-warning:`missing perftest`]
+
+- |uncheck_| uuid
+    .. seealso:: *link issue here*
+
+- |uncheck_| Conditional: If, Else, ElseIf
+    .. seealso:: *link issue here*
+
+- |check_| VarInt (C type: :c:type:`CpVarIntAtomObject`, Py type: :class:`varint_t`)
+    [:text-danger:`missing docs`],
+    [:text-warning:`missing perftest`]
+
+- |uncheck_| While
+    .. seealso:: *link issue here*