diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..76f22a726 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,2 @@ +*.gem +pkg diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..58c78a4b0 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,39 @@ +# Dependencies we need available when running +ARG RUN_DEPS="ruby-dev llvm libclang-3.8-dev python-pygments libffi6" + +FROM debian:stretch-slim as builder + +# Extra dependencies we need in the build container +ARG BUILD_DEPS="git cmake pkg-config build-essential libffi-dev libssl-dev" + +# takes the global value defined above +ARG RUN_DEPS + +# We need these packages to build ruby extensions, rugged and to parse the C +# code. pygments is there to highlight the code examples. +RUN apt update && \ + apt install -y --no-install-recommends ${BUILD_DEPS} ${RUN_DEPS} && \ + apt clean && \ + rm -rf /var/lib/apt/lists/* + +WORKDIR /docurium + +# This is here so we can provide a unique argument per run so docker does not +# consider the rest of the file cached and wealways install the latest version +# of docurium +ARG CACHEBUST=1 + +COPY . /docurium/ +RUN gem build docurium && gem install docurium-*.gem --no-ri --no-rdoc + +FROM debian:stretch-slim + +# takes the global value defined above +ARG RUN_DEPS +RUN apt update && \ + apt install -y --no-install-recommends ${RUN_DEPS} && \ + apt clean && \ + rm -rf /var/lib/apt/lists/* + +COPY --from=builder /var/lib/gems/ /var/lib/gems/ +COPY --from=builder /usr/local/bin/cm /usr/local/bin/cm diff --git a/Rakefile b/Rakefile index c7b2163e8..dec0fb2e6 100644 --- a/Rakefile +++ b/Rakefile @@ -5,10 +5,22 @@ require 'rubygems/package_task' task :default => :test gemspec = Gem::Specification::load(File.expand_path('../docurium.gemspec', __FILE__)) -Gem::PackageTask.new(gemspec) do |pkg| -end +Gem::PackageTask.new(gemspec).define Rake::TestTask.new do |t| t.libs << 'libs' << 'test' t.pattern = 'test/**/*_test.rb' end + +namespace :docker do + task :build do + sh("docker build . -t docurium") + end + + task :run, [:on] do |t, args| + src = File.realpath(args[:on]) + sh("echo \"# cd /workspace && cm doc api.docurium\"") + sh("docker run -v #{src}:/workspace -it docurium") + end +end + diff --git a/docurium.gemspec b/docurium.gemspec index ca091c7dc..ce9597aec 100644 --- a/docurium.gemspec +++ b/docurium.gemspec @@ -20,6 +20,7 @@ Gem::Specification.new do |s| s.add_dependency "rugged", "~>0.21" s.add_dependency "redcarpet", "~>3.0" s.add_dependency "ffi-clang", "~> 0.5" + s.add_dependency "parallel", "~> 1.17.0" s.add_development_dependency "rake", "~> 12" s.add_development_dependency "minitest", "~> 5.11" diff --git a/lib/docurium.rb b/lib/docurium.rb index 300e1fb60..da5d77762 100644 --- a/lib/docurium.rb +++ b/lib/docurium.rb @@ -11,6 +11,7 @@ require 'rugged' require 'redcarpet' require 'redcarpet/compat' +require 'parallel' require 'thread' # Markdown expects the old redcarpet compat API, so let's tell it what @@ -118,69 +119,41 @@ def format_examples!(data, version) def generate_doc_for(version) index = Rugged::Index.new read_subtree(index, version, option_version(version, 'input', '')) + data = parse_headers(index, version) - data + examples = format_examples!(data, version) + [data, examples] end def process_project(versions) - nversions = versions.size - output = Queue.new - pipes = {} - versions.each do |version| - # We don't need to worry about joining since this process is - # going to die immediately - read, write = IO.pipe - pid = Process.fork do - read.close - - data = generate_doc_for(version) - examples = format_examples!(data, version) - - Marshal.dump([version, data, examples], write) - write.close - end - - pipes[pid] = read - write.close - end - - print "Generating documentation [0/#{nversions}]\r" - - # This may seem odd, but we need to keep reading from the pipe or - # the buffer will fill and they'll block and never exit. Therefore - # we can't rely on Process.wait to tell us when the work is - # done. Instead read from all the pipes concurrently and send the - # ruby objects through the queue. - Thread.abort_on_exception = true - pipes.each do |pid, read| - Thread.new do - result = read.read - output << Marshal.load(result) - end - end - - for i in 1..nversions - version, data, examples = output.pop - + nversions = versions.count + Parallel.each_with_index(versions, finish: -> (version, index, result) do + data, examples = result # There's still some work we need to do serially tally_sigs!(version, data) force_utf8(data) + puts "Adding documentation for #{version} [#{index}/#{nversions}]" + # Store it so we can show it at the end @head_data = data if version == 'HEAD' - yield i, nversions, version, data, examples if block_given? + yield index, version, result if block_given? + + end) do |version, index| + puts "Generating documentation for #{version} [#{index}/#{nversions}]" + generate_doc_for(version) end end - def generate_docs(options) + def generate_docs output_index = Rugged::Index.new write_site(output_index) @tf = File.expand_path(File.join(File.dirname(__FILE__), 'docurium', 'layout.mustache')) versions = get_versions versions << 'HEAD' # If the user specified versions, validate them and overwrite - if !(vers = (options[:for] || [])).empty? + if !(vers = (@cli_options[:for] || [])).empty? vers.each do |v| next if versions.include?(v) puts "Unknown version #{v}" @@ -189,8 +162,9 @@ def generate_docs(options) versions = vers end - process_project(versions) do |i, version, data, examples| - @repo.write(data.to_json, :blob) + process_project(versions) do |i, version, result| + data, examples = result + sha = @repo.write(data.to_json, :blob) print "Generating documentation [#{i}/#{versions.count}]\r" @@ -314,7 +288,7 @@ def initialize(warning, type, identifier, opts = {}) def message msg = self._message - msg.shift % msg.map {|a| self.send(a).to_s } if msg.kind_of?(Array) + msg.kind_of?(Array) ? msg.shift % msg.map {|a| self.send(a).to_s } : msg end end @@ -457,7 +431,7 @@ def find_subtree(version, path) def read_subtree(index, version, path) tree = find_subtree(version, path) - index.read_tree(tree) + index.read_tree(tree) rescue nil end def valid_config(file) diff --git a/lib/docurium/docparser.rb b/lib/docurium/docparser.rb index 29edf05cb..1cd02d428 100644 --- a/lib/docurium/docparser.rb +++ b/lib/docurium/docparser.rb @@ -70,7 +70,15 @@ def parse_file(orig_filename, opts = {}) args = includes.map { |path| "-I#{path}" } args << '-ferror-limit=1' - tu = Index.new(true, true).parse_translation_unit(filename, args, @unsaved, {:detailed_preprocessing_record => 1}) + attempts = 3 + begin + tu = Index.new(true, true).parse_translation_unit(filename, args, @unsaved, {:detailed_preprocessing_record => 1}) + rescue Exception => e + puts "Exception raised while parsing #{filename}:#{e}" + puts e.inspect + retry if attempts > 0 + return [] + end recs = [] @@ -87,7 +95,6 @@ def parse_file(orig_filename, opts = {}) # :continue #end - next :continue if cursor.comment.kind == :comment_null next :continue if cursor.spelling == "" extent = cursor.extent @@ -111,6 +118,14 @@ def parse_file(orig_filename, opts = {}) when :cursor_typedef_decl debug "have typedef #{cursor.spelling} #{cursor.underlying_type.spelling}" rec.update extract_typedef(cursor) + when :cursor_variable + next :continue + when :cursor_macro_definition + next :continue + when :cursor_inclusion_directive + next :continue + when :cursor_macro_expansion + next :continue else raise "No idea how to deal with #{cursor.kind}" end diff --git a/test/docurium_test.rb b/test/docurium_test.rb index 0cf1ed054..6e04e9035 100644 --- a/test/docurium_test.rb +++ b/test/docurium_test.rb @@ -28,10 +28,17 @@ def teardown FileUtils.remove_entry(@dir) end + def test_can_parse + refute_nil @data + assert_equal [:files, :functions, :callbacks, :globals, :types, :prefix, :groups], @data.keys + files = %w(blob.h callback.h cherrypick.h commit.h common.h errors.h index.h object.h odb.h odb_backend.h oid.h refs.h repository.h revwalk.h signature.h tag.h tree.h types.h) + assert_equal files, @data[:files].map {|d| d[:file] } + end + def test_can_parse_headers keys = @data.keys.map { |k| k.to_s }.sort assert_equal ['callbacks', 'files', 'functions', 'globals', 'groups', 'prefix', 'types'], keys - assert_equal 155, @data[:functions].size + assert_equal 159, @data[:functions].size end def test_can_extract_enum_from_define @@ -164,6 +171,40 @@ def test_can_parse_function_cast_args assert_equal expect_comment.split("\n"), func[:comments].split("\n") end + def test_can_handle_bulleted_lists + type = @data[:types].find {|name, data| name == 'git_repository_init_options' } + refute_nil type + expect_comment = <<-EOF +

This contains extra options for git_repository_init_ext that enable + additional initialization features. The fields are:

+ + +EOF + assert_equal expect_comment, type[1][:comments] + end + def test_can_get_the_full_description_from_multi_liners func = @data[:functions]['git_commit_create_o'] desc = "

Create a new commit in the repository using git_object\n instances as parameters.

\n" @@ -171,8 +212,8 @@ def test_can_get_the_full_description_from_multi_liners end def test_can_group_functions - groups = %w(blob cherrypick commit index lasterror object odb oid reference repository revwalk signature strerror tag tree treebuilder work) - assert_equal groups, @data[:groups].map {|g| g[0]}.sort + groups = %w(blob cherrypick commit index lasterror object odb oid reference repository revwalk signature strarray strerror tag tree treebuilder work) + assert_equal groups, @data[:groups].map {|g| g[0]} group, funcs = @data[:groups].first assert_equal 'blob', group assert_equal 6, funcs.size diff --git a/test/fixtures/git2/types.h b/test/fixtures/git2/types.h index e2e2c971f..fac447573 100644 --- a/test/fixtures/git2/types.h +++ b/test/fixtures/git2/types.h @@ -50,6 +50,7 @@ GIT_BEGIN_DECL * stat() functions, for all platforms. */ #include +#include #if defined(_MSC_VER) diff --git a/test/parser_test.rb b/test/parser_test.rb index 72793fac9..52c327dc2 100644 --- a/test/parser_test.rb +++ b/test/parser_test.rb @@ -172,29 +172,81 @@ def test_return_struct EOF actual = parse(name, contents) - expected = [{ - :file => "tree.h", - :line => 7, - :lineto => 7, - :tdef => nil, - :type => :function, - :name => "git_tree_owner", - :body => "git_repository * git_tree_owner(git_tree *tree);", - :description => " Weak owner ref", - :comments => "", - :sig => "git_tree *", - :args => [{ - :name => "tree", - :type => "git_tree *", - :comment => nil - }], - :return => { - :type => "git_repository *", - :comment => nil + expected = [ + { + :file => "tree.h", + :line => 1, + :lineto => 1, + :tdef => nil, + :type => :struct, + :name => "git_repository", + :description => "", + :comments => "", + :fields => [], + :decl => [] }, - :decl => "git_repository * git_tree_owner(git_tree *tree)", - :argline => "git_tree *tree" - }] + { + :file => "tree.h", + :line => 1, + :lineto => 1, + :tdef => :typedef, + :name => "git_repository", + :underlying_type => "struct git_repository", + :type => :struct, + :decl => "git_repository", + :description => "", + :comments => "" + }, + { + :file => "tree.h", + :line => 2, + :lineto => 2, + :tdef => nil, + :type => :struct, + :name => "git_tree", + :description => "", + :comments => "", + :fields => [], + :decl => [] + }, + { + :file => "tree.h", + :line => 2, + :lineto => 2, + :tdef => :typedef, + :name => "git_tree", + :underlying_type => "struct git_tree", + :type => :struct, + :decl => "git_tree", + :description => "", + :comments => "" + }, + { + :file => "tree.h", + :line => 7, + :lineto => 7, + :tdef => nil, + :type => :function, + :name => "git_tree_owner", + :body => "git_repository * git_tree_owner(git_tree *tree);", + :description => " Weak owner ref", + :comments => "", + :sig => "git_tree *", + :args => [ + { + :name => "tree", + :type => "git_tree *", + :comment => nil + } + ], + :return => { + :type => "git_repository *", + :comment => nil + }, + :decl => "git_repository * git_tree_owner(git_tree *tree)", + :argline => "git_tree *tree" + } + ] assert_equal expected, actual @@ -392,7 +444,20 @@ def test_type_reference EOF actual = parse(name, contents) - expected = [{ + expected = [ + { + :file => "typeref.h", + :line => 4, + :lineto => 4, + :tdef => nil, + :type => :struct, + :name => "my_type", + :description => "", + :comments => "", + :fields => [], + :decl => [] + }, + { :file => "typeref.h", :line => 4, :lineto => 4,