Restructure

.github/workflows/cpp-bindings.yml

@@ -29,6 +29,6 @@ jobs:
       - name: Compile prism
         run: bundle exec rake compile
       - name: Compile C++
-        run: g++ -o ./cpp_test cpp/test.cpp build/static/*.o build/static/util/*.o -Iinclude
+        run: g++ -o ./cpp_test cpp/test.cpp build/static/*.o -Iinclude
       - name: Run C++
         run: ./cpp_test

.gitignore

@@ -31,8 +31,7 @@ out.svg
 /fuzz/output/
 /gemfiles/typecheck/bin/
 /include/prism/ast.h
-/include/prism/diagnostic.h
-/include/prism/node_new.h
+/include/prism/internal/diagnostic.h
 /javascript/node_modules/
 /javascript/package-lock.json
 /javascript/src/deserialize.js
@@ -58,7 +57,7 @@ out.svg
 /src/node.c
 /src/prettyprint.c
 /src/serialize.c
-/src/token_type.c
+/src/tokens.c
 /src/**/*.o
 /rbi/prism/dsl.rbi
 /rbi/prism/node.rbi

Doxyfile

@@ -23,8 +23,8 @@ PROJECT_NAME           = "Prism Ruby parser"
 OUTPUT_DIRECTORY       = doc
 JAVADOC_AUTOBRIEF      = YES
 OPTIMIZE_OUTPUT_FOR_C  = YES
-INPUT                  = src src/util include include/prism include/prism/util
-EXCLUDE                = include/prism/debug_allocator.h
+INPUT                  = include/prism.h include/prism
+EXCLUDE                = include/prism/internal
 HTML_OUTPUT            = c
 SORT_MEMBER_DOCS       = NO
 GENERATE_LATEX         = NO

Makefile

@@ -70,12 +70,12 @@ build/fuzz.%: $(SOURCES) fuzz/%.c fuzz/fuzz.c
 	$(ECHO) "building $* fuzzer"
 	$(Q) $(MAKEDIRS) $(@D)
 	$(ECHO) "building main fuzz binary"
-	$(Q) afl-clang-lto $(DEBUG_FLAGS) $(CPPFLAGS) $(CFLAGS) $(FUZZ_FLAGS) -O0 -fsanitize-ignorelist=fuzz/asan.ignore -fsanitize=fuzzer,address -ggdb3 -std=c99 -Iinclude -o $@ $^
+	$(Q) afl-clang-lto $(DEBUG_FLAGS) $(CPPFLAGS) $(CFLAGS) $(FUZZ_FLAGS) -O0 -fsanitize=fuzzer,address -ggdb3 -std=c99 -Iinclude -o $@ $^
 	$(ECHO) "building cmplog binary"
-	$(Q) AFL_LLVM_CMPLOG=1 afl-clang-lto $(DEBUG_FLAGS) $(CPPFLAGS) $(CFLAGS) $(FUZZ_FLAGS) -O0 -fsanitize-ignorelist=fuzz/asan.ignore -fsanitize=fuzzer,address -ggdb3 -std=c99 -Iinclude -o $@.cmplog $^
+	$(Q) AFL_LLVM_CMPLOG=1 afl-clang-lto $(DEBUG_FLAGS) $(CPPFLAGS) $(CFLAGS) $(FUZZ_FLAGS) -O0 -fsanitize=fuzzer,address -ggdb3 -std=c99 -Iinclude -o $@.cmplog $^
 
 build/fuzz.heisenbug.%: $(SOURCES) fuzz/%.c fuzz/heisenbug.c
-	$(Q) afl-clang-lto $(DEBUG_FLAGS) $(CPPFLAGS) $(CFLAGS) $(FUZZ_FLAGS) -O0 -fsanitize-ignorelist=fuzz/asan.ignore -fsanitize=fuzzer,address -ggdb3 -std=c99 -Iinclude -o $@ $^
+	$(Q) afl-clang-lto $(DEBUG_FLAGS) $(CPPFLAGS) $(CFLAGS) $(FUZZ_FLAGS) -O0 -fsanitize=fuzzer,address -ggdb3 -std=c99 -Iinclude -o $@ $^
 
 fuzz-debug:
 	$(ECHO) "entering debug shell"

README.md

@@ -44,7 +44,6 @@ The repository contains the infrastructure for both a shared library (libprism)
 │   └── prism             Sample code that uses the Ruby API for documentation purposes
 ├── sig                   RBS type signatures for the Ruby library
 ├── src
-│   ├── util              various utility files
 │   └── prism.c           main entrypoint for the shared library
 ├── templates             contains ERB templates generated by templates/template.rb
 │   └── template.rb       generates code from the nodes and tokens configured by config.yml

cpp/test.cpp

@@ -5,21 +5,20 @@ extern "C" {
 #include <iostream>
 
 int main() {
-    pm_arena_t arena = { 0 };
-    pm_parser_t parser;
-    pm_parser_init(&arena, &parser, reinterpret_cast<const uint8_t *>("1 + 2"), 5, NULL);
+    pm_arena_t *arena = pm_arena_new();
+    pm_parser_t *parser = pm_parser_new(arena, reinterpret_cast<const uint8_t *>("1 + 2"), 5, NULL);
 
-    pm_node_t *root = pm_parse(&parser);
-    pm_buffer_t buffer = { 0 };
+    pm_node_t *root = pm_parse(parser);
+    pm_buffer_t *buffer = pm_buffer_new();
 
-    pm_prettyprint(&buffer, &parser, root);
-    pm_buffer_append_byte(&buffer, '\0');
+    pm_prettyprint(buffer, parser, root);
 
-    std::cout << buffer.value << std::endl;
+    std::string_view view(pm_buffer_value(buffer), pm_buffer_length(buffer));
+    std::cout << view << std::endl;
 
-    pm_buffer_free(&buffer);
-    pm_parser_free(&parser);
-    pm_arena_free(&arena);
+    pm_buffer_free(buffer);
+    pm_parser_free(parser);
+    pm_arena_free(arena);
 
     return 0;
 }

docs/build_system.md

@@ -87,7 +87,7 @@ If you need to use memory allocation functions implemented outside of the standa
 * Additionally, include `-I [path/to/custom_allocator]` where your `prism_xallocator.h` is located
 * Link the implementation of `prism_xallocator.c` that contains functions declared in `prism_xallocator.h`
 
-For further clarity, refer to `include/prism/defines.h`.
+For further clarity, refer to `include/prism/internal/allocator.h`.
 
 ### Building prism from source as a C library
 

docs/encoding.md

@@ -107,7 +107,7 @@ For each of these encodings, prism provides functions for checking if the subseq
 
 ## Getting notified when the encoding changes
 
-You may want to get notified when the encoding changes based on the result of parsing an encoding comment. We use this internally for our `lex` function in order to provide the correct encodings for the tokens that are returned. For that you can register a callback with `pm_parser_register_encoding_changed_callback`. The callback will be called with a pointer to the parser. The encoding can be accessed through `parser->encoding`.
+You may want to get notified when the encoding changes based on the result of parsing an encoding comment. We use this internally for our `lex` function in order to provide the correct encodings for the tokens that are returned. For that you can register a callback with `pm_parser_encoding_changed_callback_set`. The callback will be called with a pointer to the parser. The encoding can be accessed through `parser->encoding`.
 
 ```c
 // When the encoding that is being used to parse the source is changed by prism,
@@ -117,5 +117,5 @@ typedef void (*pm_encoding_changed_callback_t)(pm_parser_t *parser);
 // Register a callback that will be called whenever prism changes the encoding
 // it is using to parse based on the magic comment.
 PRISM_EXPORTED_FUNCTION void
-pm_parser_register_encoding_changed_callback(pm_parser_t *parser, pm_encoding_changed_callback_t callback);
+pm_parser_encoding_changed_callback_set(pm_parser_t *parser, pm_encoding_changed_callback_t callback);
 ```

docs/fuzzing.md

@@ -5,34 +5,29 @@ We use fuzzing to test the various entrypoints to the library. The fuzzer we use
 ```
 fuzz
 ├── corpus
-│   ├── parse             fuzzing corpus for parsing (a symlink to our fixtures)
-│   └── regexp            fuzzing corpus for regexp
+│   └── parse             fuzzing corpus for parsing (a symlink to our fixtures)
 ├── dict                  a AFL++ dictionary containing various tokens
 ├── docker
 │   └── Dockerfile        for building a container with the fuzzer toolchain
 ├── fuzz.c                generic entrypoint for fuzzing
 ├── heisenbug.c           entrypoint for reproducing a crash or hang
 ├── parse.c               fuzz handler for parsing
 ├── parse.sh              script to run parsing fuzzer
-├── regexp.c              fuzz handler for regular expression parsing
-├── regexp.sh             script to run regexp fuzzer
 └── tools
     ├── backtrace.sh      generates backtrace files for a crash directory
     └── minimize.sh       generates minimized crash or hang files
 ```
 
 ## Usage
 
-There are currently three fuzzing targets
+There is currently one fuzz target:
 
 - `pm_serialize_parse` (parse)
-- `pm_regexp_parse` (regexp)
 
-Respectively, fuzzing can be performed with
+Fuzzing can be performed with
 
 ```
 make fuzz-run-parse
-make fuzz-run-regexp
 ```
 
 To end a fuzzing job, interrupt with CTRL+C. To enter a container with the fuzzing toolchain and debug utilities, run
@@ -43,8 +38,6 @@ make fuzz-debug
 
 # Out-of-bounds reads
 
-Currently, encoding functionality implementing the `pm_encoding_t` interface can read outside of inputs. For the time being, ASAN instrumentation is disabled for functions from src/enc. See `fuzz/asan.ignore`.
-
 To disable ASAN read instrumentation globally, use the `FUZZ_FLAGS` environment variable e.g.
 
 ```
@@ -55,7 +48,7 @@ Note, that this may make reproducing bugs difficult as they may depend on memory
 
 ```
 make fuzz-debug # enter the docker container with build tools
-make build/fuzz.heisenbug.parse # or .regexp
+make build/fuzz.heisenbug.parse
 ./build/fuzz.heisenbug.parse path-to-problem-input
 ```
 

docs/serialization.md

@@ -159,8 +159,8 @@ typedef struct {
   size_t capacity;
 } pm_buffer_t;
 
-// Free the memory associated with the buffer.
-void pm_buffer_free(pm_buffer_t *);
+// Free the memory held by the buffer.
+void pm_buffer_cleanup(pm_buffer_t *);
 
 // Parse and serialize the AST represented by the given source to the given
 // buffer.
@@ -172,12 +172,12 @@ Typically you would use a stack-allocated `pm_buffer_t` and call `pm_serialize_p
 ```c
 void
 serialize(const uint8_t *source, size_t length) {
-  pm_buffer_t buffer = { 0 };
-  pm_serialize_parse(&buffer, source, length, NULL);
+  pm_buffer_t *buffer = pm_buffer_new();
+  pm_serialize_parse(buffer, source, length, NULL);
 
   // Do something with the serialized string.
 
-  pm_buffer_free(&buffer);
+  pm_buffer_free(buffer);
 }
 ```
 

ext/prism/extconf.rb

@@ -118,10 +118,7 @@ def add_libprism_source(path)
   src_list path
 end
 
-$srcs = src_list("$(srcdir)") +
-  add_libprism_source("$(srcdir)/../../src") +
-  add_libprism_source("$(srcdir)/../../src/util")
-
+$srcs = src_list("$(srcdir)") + add_libprism_source("$(srcdir)/../../src")
 $headers += Dir["#{$srcdir}/../../include/**/*.h"]
 
 # Finally, we'll create the `Makefile` that is going to be used to configure and