diff --git a/.eslintrc.js b/.eslintrc.js new file mode 100644 index 0000000000..1561b367b1 --- /dev/null +++ b/.eslintrc.js @@ -0,0 +1,71 @@ +module.exports = { + "env": { + "browser": true, + "es6": true, + "node": true, + "mocha": true + }, + "extends": [ + "eslint:recommended", + "standard" + ], + "globals": { + "Atomics": "readonly", + "SharedArrayBuffer": "readonly" + }, + "parserOptions": { + "ecmaVersion": 2018, + "sourceType": "module" + }, + "parser": '@typescript-eslint/parser', + "plugins": [ + "@typescript-eslint" + ], + "rules": { + "array-callback-return": "error", + "block-scoped-var": "error", + // we like our semi-colons + "semi": ["error","always"], + // our codebase doesn't do this at all, so disabled for now + "space-before-function-paren": ["error","never"], + // for now ignore diff between types of quoting + "quotes": "off", + // this is the style we are already using + "operator-linebreak": ["error","before", { "overrides": { "?": "after", ":": "after", "+": "after" } }], + // sometimes we declare variables with extra spacing + "indent": ["error", 2, {"VariableDeclarator":2}], + // seems like a good idea not to use explicit undefined + "no-undefined": "error", + // ensure import specifier contains file extension + "import/extensions": ["error", "always"], + + // TODO maybe + "camelcase": "off", // TODO: turn on later + "init-declarations": ["error","always"] + }, + "overrides": [ + { + "files": ["src/**/*.js"], + "rules": { + // make sure there is no Node.js specific API slipping into the source files + "import/no-nodejs-modules": "error", + "import/no-commonjs": "error", + } + }, + { + "files": ["src/languages/*.js"], + "rules": { + "no-unused-expressions": "off", + // languages are all over the map and we don't want to + // do a mass edit so turn off the most egregious rule violations + "indent": "off", + "comma-dangle": "off", + "array-bracket-spacing": "off", + "object-curly-spacing": "off", + "key-spacing": "off", + "object-curly-newline": "off", + "object-property-newline": "off" + } + } + ] +}; diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000000..9bb5ffd29d --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,20 @@ +--- +name: Feature request +about: Suggest a new idea or feature... +title: "[Request] ..." +labels: enhancement +assignees: '' + +--- + +**Is your request related to a specific problem you're having?** +A clear and concise description of the problem itself. Ex. I'm always frustrated when [...] + +**The solution you'd prefer / feature you'd like to see added...** +A clear and concise description of how you imagine we might solve this (if you have any ideas). + +**Any alternative solutions you considered...** +A clear and concise description of any alternative solutions or features you may have considered. + +**Additional context...** +Add any other context or screenshots relating to the feature request here. diff --git a/.github/ISSUE_TEMPLATE/incorrect-syntax-highlighting.md b/.github/ISSUE_TEMPLATE/incorrect-syntax-highlighting.md new file mode 100644 index 0000000000..f9fa81ebf1 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/incorrect-syntax-highlighting.md @@ -0,0 +1,39 @@ +--- +name: Incorrect syntax highlighting +about: Report incorrect syntax highlighting... +title: "(language name) Short description of issue..." +labels: bug, help welcome, language +assignees: '' + +--- + +**Describe the issue** + + +**Which language seems to have the issue?** + + +**Are you using `highlight` or `highlightAuto`?** + +... + +**Sample Code to Reproduce** + + + +**Expected behavior** + + +**Additional context** + diff --git a/.github/ISSUE_TEMPLATE/language-request.md b/.github/ISSUE_TEMPLATE/language-request.md new file mode 100644 index 0000000000..f8d55e1335 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/language-request.md @@ -0,0 +1,42 @@ +--- +name: Language request +about: I really wish Highlight.js could highlight ... +title: PLEASE read the below carefully. +labels: '' +assignees: '' + +--- + +First let us say that we'd also love it if Highlight.js could highlight whichever language you're about to request support for! And there is a chance you can help make that happen! But first... + +...PLEASE READ THE FOLLOWING... + + Highlight.js does not have a fundamental plan for implementing new languages + - i.e., the core team doesn't usually develop new languages. The core team + instead focuses on parser development, bugs, and supporting the existing + languages. They also currently does not have time to review, merge and + maintain any additional languages (fixing bugs, dealing with issues, etc). + + Instead, the project works by encouraging 3rd party language grammars from + contributors willing to help develop and maintain them. We're also happy to + host those 3rd party language grammars at the ``highlightjs`` GitHub + organization - no matter how obscure or weird. Or you're wlecome to host it + yourself - we're still happy to link to it. + + This means that *there's no point in requesting a new language without also + providing a 3rd party implementation* (we'll simply close "Please support + language Xyz" issues with a link to this explanation). If you'd like to see + a particular language available but cannot implement it, the best way to + make it happen is to find another developer interested in doing so. + + For more info on actually developing a language see our :doc:`language-guide`, + and for information on how to properly package your 3rd party language module + see :doc:`language-contribution`. + +If you are interested in contributing a 3rd party language grammar you can start with: + +- https://highlightjs.readthedocs.io/en/latest/language-contribution.html + +--- + +You actually don't need to create this issue at all unless you have a specific question about the 3rd party language grammar creation process - which we'd be glad to answer. diff --git a/.gitignore b/.gitignore index 033722def3..52ed616381 100644 --- a/.gitignore +++ b/.gitignore @@ -6,7 +6,12 @@ __pycache__ node_modules .project yarn.lock +extra/ # editors .idea/ .vscode/ +.Rproj.user + +# misc +/work diff --git a/.jshintrc b/.jshintrc new file mode 100644 index 0000000000..573812f43d --- /dev/null +++ b/.jshintrc @@ -0,0 +1,7 @@ +// whole codebase isn't ES8/9 yet, but our tests and some things are +{ + "esversion": 9, + "node": true, + // eslint warns us about semicolons + "-W033": false +} diff --git a/.travis.yml b/.travis.yml index 7ef674fb0d..67fd1364cf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,28 +3,33 @@ node_js: - "lts/*" - "node" env: - - - - BROWSER=1 - - BROWSER=1 NOCOMPRESS=1 + # we current only test "use strict" for our NPM builds + - BUILD=node TEST_STRICT_BUNDLE=1 + - BUILD=browser + - BUILD=browser NO_MINIFY=1 script: - | - export BUILD_PARAMS="" + export BUILD_PARAMS="-t $BUILD" - if [ "x$BROWSER" = "x1" ]; then - export BUILD_PARAMS="$BUILD_PARAMS -t browser" - else - export BUILD_PARAMS="$BUILD_PARAMS -t node" - fi - - if [ "x$NOCOMPRESS" = "x1" ]; then + if [ "x$NO_MINIFY" = "x1" ]; then export BUILD_PARAMS="$BUILD_PARAMS -n" fi node tools/build.js $BUILD_PARAMS - if [ "x$BROWSER" = "x1" ]; then - npm run test-browser - else + # test that our build is "use strict" safe for use with packaging + # systems importing our source thru ES6 modules (rollup, etc.) + if [ "x$TEST_STRICT_BUNDLE" = "x1" ]; then + ./node_modules/.bin/rollup -c test/builds/rollup_import_via_commonjs.js + node build/bundle.js || exit 1 + rm build/bundle.js + fi + + if [ "x$BUILD" = "xnode" ]; then npm run test + else + npm run test-browser + # our browser build should also work fine as a Node.js CommonJS module + node test/builds/browser_build_as_commonjs.js fi sudo: false # Use container-based architecture diff --git a/AUTHORS.en.txt b/AUTHORS.txt similarity index 97% rename from AUTHORS.en.txt rename to AUTHORS.txt index 638ecc7ec9..131f873ad8 100644 --- a/AUTHORS.en.txt +++ b/AUTHORS.txt @@ -9,6 +9,7 @@ Current core developers (alphabetical): - Li Xuanji - Marcos Cáceres - Sang Dang +- Josh Goebel Former maintainers: @@ -19,6 +20,7 @@ Former maintainers: Contributors: - Peter Leonov +- Vanessa Sochat <@vsoch> - Victor Karamzin - Vsevolod Solovyov - Anton Kovalyov @@ -270,6 +272,7 @@ Contributors: - Harmon - Eric Bailey - Gustavo Costa +- Jeffrey Arnold - Antoine Boisier-Michaud - Alejandro Isaza - Laurent Voullemier @@ -280,4 +283,8 @@ Contributors: - Yuri Mazursky - Carl Baxter - Thomas Reichel - +- G8t Guy +- Samia Ali +- Alexandre Grison +- Jim Mason +- lioshi diff --git a/CHANGES.md b/CHANGES.md index 018992dfe3..34a5e311b3 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,4 +1,322 @@ -## Version 9.16.0 +## Version 10.1.2 + +Fixes: + +- fix(night) Prevent object prototype values from being returned by `getLanguage` (#2636) [night][] + +[night]: https://github.com/night + + +## Version 10.1.1 + +Fixes: + +- Resolve issue on Node 6 due to dangling comma (#2608) [Edwin Hoogerbeets][] +- Resolve `index.d.ts is not a module` error (#2603) [Josh Goebel][] + +[Josh Goebel]: https://github.com/yyyc514 +[Edwin Hoogerbeets]: https://github.com/ehoogerbeets + + +## Version 10.1.0 + +New themes: + +- *NNFX* and *NNFX-dark* by [Jim Mason][] +- *lioshi* by [lioshi][] + +Parser Engine: + +- (parser) Now escapes quotes in text content when escaping HTML (#2564) [Josh Goebel][] +- (parser) Adds `keywords.$pattern` key to grammar definitions (#2519) [Josh Goebel][] +- (parser) Adds SHEBANG utility mode [Josh Goebel][] +- (parser) Adds `registerAliases` method (#2540) [Taufik Nurrohman][] +- (enh) Added `on:begin` callback for modes (#2261) [Josh Goebel][] +- (enh) Added `on:end` callback for modes (#2261) [Josh Goebel][] +- (enh) Added ability to programatically ignore begin and end matches (#2261) [Josh Goebel][] +- (enh) Added `END_SAME_AS_BEGIN` mode to replace `endSameAsBegin` parser attribute (#2261) [Josh Goebel][] +- (fix) `fixMarkup` would rarely destroy markup when `useBR` was enabled (#2532) [Josh Goebel][] + +Deprecations: + +- `htmlbars` grammar is now deprecated. Use `handlebars` instead. (#2344) [Nils Knappmeier][] +- when using `highlightBlock` `result.re` deprecated. Use `result.relevance` instead. (#2552) [Josh Goebel][] +- ditto for `result.second_best.re` => `result.second_best.relevance` (#2552) +- `lexemes` is now deprecated in favor of `keywords.$pattern` key (#2519) [Josh Goebel][] +- `endSameAsBegin` is now deprecated. (#2261) [Josh Goebel][] + +Language Improvements: + +- fix(groovy) strings are not allowed inside ternary clauses (#2217) [Josh Goebel][] +- fix(typescript) add `readonly` keyword (#2562) [Martin (Lhoerion)][] +- fix(javascript) fix regex inside parens after a non-regex (#2530) [Josh Goebel][] +- enh(typescript) use identifier to match potential keywords, preventing false positivites (#2519) [Josh Goebel][] +- enh(javascript) use identifier to match potential keywords, preventing false positivites (#2519) [Josh Goebel][] +- [enh] Add `OPTIMIZE:` and `HACK:` to the labels highlighted inside comments [Josh Goebel][] +- enh(typescript/javascript/coffeescript/livescript) derive ECMAscript keywords from a common foudation (#2518) [Josh Goebel][] +- enh(typescript) add setInterval, setTimeout, clearInterval, clearTimeout (#2514) [Josh Goebel][] +- enh(javascript) add setInterval, setTimeout, clearInterval, clearTimeout (#2514) [Vania Kucher][] +- enh(cpp) add `pair`, `make_pair`, `priority_queue` as built-ins (#2538) [Hankun Lin][] +- enh(cpp) recognize `priority_queue` `pair` as cpp containers (#2541) [Hankun Lin][] +- fix(javascript) prevent `set` keyword conflicting with setTimeout, etc. (#2514) [Vania Kucher][] +- fix(cpp) Fix highlighting of unterminated raw strings (#2261) [David Benjamin][] +- fix(javascript) `=>` function with nested `()` in params now works (#2502) [Josh Goebel][] +- fix(typescript) `=>` function with nested `()` in params now works (#2502) [Josh Goebel][] +- fix(yaml) Fix tags to include non-word characters (#2486) [Peter Plantinga][] +- fix(swift) `@objcMembers` was being partially highlighted (#2543) [Nick Randall][] +- enh(dart) Add `late` and `required` keywords, the `Never` built-in type, and nullable built-in types (#2550) [Sam Rawlins][] +- enh(erlang) Add underscore separators to numeric literals (#2554) [Sergey Prokhorov][] +- enh(handlebars) Support for sub-expressions, path-expressions, hashes, block-parameters and literals (#2344) [Nils Knappmeier][] +- enh(protobuf) Support multiline comments (#2597) [Pavel Evstigneev][] +- fix(toml) Improve key parsing (#2595) [Antoine du Hamel][] + +[Josh Goebel]: https://github.com/yyyc514 +[Peter Plantinga]: https://github.com/pplantinga +[David Benjamin]: https://github.com/davidben +[Vania Kucher]: https://github.com/qWici +[Hankun Lin]: https://github.com/Linhk1606 +[Nick Randall]: https://github.com/nicked +[Sam Rawlins]: https://github.com/srawlins +[Sergey Prokhorov]: https://github.com/seriyps +[Nils Knappmeier]: https://github.com/nknapp +[Martin (Lhoerion)]: https://github.com/Lhoerion +[Jim Mason]: https://github.com/RocketMan +[lioshi]: https://github.com/lioshi +[Pavel Evstigneev]: https://github.com/Paxa +[Antoine du Hamel]: https://github.com/aduh95 + + +## Version 10.0.2 + +Brower build: + +- [Issue](https://github.com/highlightjs/highlight.js/issues/2505) (bug) Fix: Version 10 fails to load as CommonJS module. (#2511) [Josh Goebel][] +- [Issue](https://github.com/highlightjs/highlight.js/issues/2505) (removal) AMD module loading support has been removed. (#2511) [Josh Goebel][] + +Parser Engine Changes: + +- [Issue](https://github.com/highlightjs/highlight.js/issues/2522) fix(parser) Fix freez issue with illegal 0 width matches (#2524) [Josh Goebel][] + + +[Josh Goebel]: https://github.com/yyyc514 + + +## Version 10.0.1 + +Parser Engine Changes: + +- (bug) Fix sublanguage with no relevance score (#2506) [Josh Goebel][] + +[Josh Goebel]: https://github.com/yyyc514 + + +## Version 10.0.0 + +New languages: + +- add(php-template) Explicit language to detect PHP templates (vs xml) [Josh Goebel][] +- enh(python) Added `python-repl` for Python REPL sessions +- add(never) Added 3rd party Never language support + +New themes: + +- *Srcery* by [Chen Bin][] + +Parser Engine Changes: + +- (bug) Fix `beginKeywords` to ignore . matches (#2434) [Josh Goebel][] +- (enh) add `before:highlight` plugin API callback (#2395) [Josh Goebel][] +- (enh) add `after:highlight` plugin API callback (#2395) [Josh Goebel][] +- (enh) split out parse tree generation and HTML rendering concerns (#2404) [Josh Goebel][] +- (enh) every language can have a `name` attribute now (#2400) [Josh Goebel][] +- (enh) improve regular expression detect (less false-positives) (#2380) [Josh Goebel][] +- (enh) make `noHighlightRe` and `languagePrefixRe` configurable (#2374) [Josh Goebel][] + +Language Improvements: + +- enh(python) Exclude parens from functions params (#2490) [Álvaro Mondéjar][] +- enh(swift) Add `compactMap` to keywords as built_in (#2478) [Omid Golparvar][] +- enh(nim) adds `func` keyword (#2468) [Adnan Yaqoob][] +- enh(xml) deprecate ActionScript inside script tags (#2444) [Josh Goebel][] +- fix(javascript) prevent get/set variables conflicting with keywords (#2440) [Josh Goebel][] +- bug(clojure) Now highlights `defn-` properly (#2438) [Josh Goebel][] +- enh(bash) default value is another variable (#2439) [Josh Goebel][] +- enh(bash) string nested within string (#2439) [Josh Goebel][] +- enh(bash) Add arithmetic expression support (#2439) [Josh Goebel][] +- enh(clojure) Add support for global definitions name (#2347) [Alexandre Grison][] +- enh(fortran) Support Fortran 77 style comments (#2416) [Josh Goebel][] +- (csharp) add support for `@identifier` style identifiers (#2414) [Josh Goebel][] +- fix(elixir) Support function names with a slash (#2406) [Josh Goebel][] +- fix(javascript) comma is allowed in a "value container" (#2403) [Josh Goebel][] +- enh(apache) add `deny` and `allow` keywords [Josh Goebel][] +- enh(apache) highlight numeric attributes values [Josh Goebel][] +- enh(apache) highlight IP addresses, ports, and strings in sections [Josh Goebel][] +- enh(php) added more keywords and include ` +https://cdn.jsdelivr.net/gh/highlightjs/cdn-release@latest/build/ + +**If you just want a single .js file with the common languages built-in: +** + +--- + +## Highlight.js + +Highlight.js is a syntax highlighter written in JavaScript. It works in +the browser as well as on the server. It works with pretty much any +markup, doesn’t depend on any framework, and has automatic language +detection. + +If you'd like to read the full README:
+ + +## License + +Highlight.js is released under the BSD License. See [LICENSE][7] file +for details. + +## Links + +The official site for the library is at . + +The Github project may be found at: + +Further in-depth documentation for the API and other topics is at +. + +Authors and contributors are listed in the [AUTHORS.txt][8] file. + +[1]: https://www.npmjs.com/package/highlight.js +[7]: https://github.com/highlightjs/highlight.js/blob/master/LICENSE +[8]: https://github.com/highlightjs/highlight.js/blob/master/AUTHORS.txt diff --git a/README.md b/README.md index 9bcafb3d59..03b249a37e 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,20 @@ # Highlight.js -[![Build Status](https://travis-ci.org/highlightjs/highlight.js.svg?branch=master)](https://travis-ci.org/highlightjs/highlight.js) [![Greenkeeper badge](https://badges.greenkeeper.io/highlightjs/highlight.js.svg)](https://greenkeeper.io/) +[![Build Status](https://travis-ci.org/highlightjs/highlight.js.svg?branch=master)](https://travis-ci.org/highlightjs/highlight.js) [![Greenkeeper badge](https://badges.greenkeeper.io/highlightjs/highlight.js.svg)](https://greenkeeper.io/) [![install size](https://packagephobia.now.sh/badge?p=highlight.js)](https://packagephobia.now.sh/result?p=highlight.js) Highlight.js is a syntax highlighter written in JavaScript. It works in the browser as well as on the server. It works with pretty much any markup, doesn’t depend on any framework, and has automatic language detection. +## Upgrading from Version 9 + +Version 10 is one of the biggest releases in quite some time. If you're +upgrading from version 9, there are some breaking changes and things you may +want to double check first. + +Please read [VERSION_10_UPGRADE.md](https://github.com/highlightjs/highlight.js/blob/master/VERSION_10_UPGRADE.md) for high-level summary of breaking changes and any actions you may need to take. See [VERSION_10_BREAKING_CHANGES.md](https://github.com/highlightjs/highlight.js/blob/master/VERSION_10_BREAKING_CHANGES.md) for a more detailed list and [CHANGES.md](https://github.com/highlightjs/highlight.js/blob/master/CHANGES.md) to learn what else is new. + ## Getting Started The bare minimum for using highlight.js on a web page is linking to the @@ -15,7 +23,7 @@ library along with one of the styles and calling ```html - + ``` @@ -27,208 +35,33 @@ work for you, you can specify the language in the `class` attribute: 
...
``` -
-The list of supported languages and corresponding classes. - -| Language | Classes | Package | -| :-----------------------| :--------------------- | :------ | -| 1C | 1c | | -| ABNF | abnf | | -| Access logs | accesslog | | -| Ada | ada | | -| ARM assembler | armasm, arm | | -| AVR assembler | avrasm | | -| ActionScript | actionscript, as | | -| Alan | alan, i | [highlightjs-alan](https://github.com/highlightjs/highlightjs-alan) | -| AngelScript | angelscript, asc | | -| Apache | apache, apacheconf | | -| AppleScript | applescript, osascript | | -| Arcade | arcade | | -| AsciiDoc | asciidoc, adoc | | -| AspectJ | aspectj | | -| AutoHotkey | autohotkey | | -| AutoIt | autoit | | -| Awk | awk, mawk, nawk, gawk | | -| Axapta | axapta | | -| Bash | bash, sh, zsh | | -| Basic | basic | | -| BNF | bnf | | -| Brainfuck | brainfuck, bf | | -| C# | cs, csharp | | -| C++ | cpp, c, cc, h, c++, h++, hpp | | -| C/AL | cal | | -| Cache Object Script | cos, cls | | -| CMake | cmake, cmake.in | | -| Coq | coq | | -| CSP | csp | | -| CSS | css | | -| Cap’n Proto | capnproto, capnp | | -| Clojure | clojure, clj | | -| CoffeeScript | coffeescript, coffee, cson, iced | | -| Crmsh | crmsh, crm, pcmk | | -| Crystal | crystal, cr | | -| Cypher (Neo4j) | cypher | [highlightjs-cypher](https://github.com/highlightjs/highlightjs-cypher) | -| D | d | | -| DNS Zone file | dns, zone, bind | | -| DOS | dos, bat, cmd | | -| Dart | dart | | -| Delphi | delphi, dpr, dfm, pas, pascal, freepascal, lazarus, lpr, lfm | | -| Diff | diff, patch | | -| Django | django, jinja | | -| Dockerfile | dockerfile, docker | | -| dsconfig | dsconfig | | -| DTS (Device Tree) | dts | | -| Dust | dust, dst | | -| Dylan | dylan | [highlight-dylan](https://github.com/highlightjs/highlight-dylan) | -| EBNF | ebnf | | -| Elixir | elixir | | -| Elm | elm | | -| Erlang | erlang, erl | | -| Excel | excel, xls, xlsx | | -| Extempore | extempore, xtlang, xtm | [highlightjs-xtlang](https://github.com/highlightjs/highlightjs-xtlang) | -| F# | fsharp, fs | | -| FIX | fix | | -| Fortran | fortran, f90, f95 | | -| G-Code | gcode, nc | | -| Gams | gams, gms | | -| GAUSS | gauss, gss | | -| GDScript | godot, gdscript | [highlightjs-gdscript](https://github.com/highlightjs/highlightjs-gdscript) | -| Gherkin | gherkin | | -| GN for Ninja | gn, gni | [highlightjs-GN](https://github.com/highlightjs/highlightjs-GN/blob/master/gn.js) | -| Go | go, golang | | -| Golo | golo, gololang | | -| Gradle | gradle | | -| Groovy | groovy | | -| HTML, XML | xml, html, xhtml, rss, atom, xjb, xsd, xsl, plist, svg | | -| HTTP | http, https | | -| Haml | haml | | -| Handlebars | handlebars, hbs, html.hbs, html.handlebars | | -| Haskell | haskell, hs | | -| Haxe | haxe, hx | | -| Hy | hy, hylang | | -| Ini, TOML | ini, toml | | -| Inform7 | inform7, i7 | | -| IRPF90 | irpf90 | | -| JSON | json | | -| Java | java, jsp | | -| JavaScript | javascript, js, jsx | | -| Kotlin | kotlin, kt | | -| Leaf | leaf | | -| Lasso | lasso, ls, lassoscript | | -| Less | less | | -| LDIF | ldif | | -| Lisp | lisp | | -| LiveCode Server | livecodeserver | | -| LiveScript | livescript, ls | | -| Lua | lua | | -| Makefile | makefile, mk, mak | | -| Markdown | markdown, md, mkdown, mkd | | -| Mathematica | mathematica, mma, wl | | -| Matlab | matlab | | -| Maxima | maxima | | -| Maya Embedded Language | mel | | -| Mercury | mercury | | -| mIRC Scripting Language | mirc, mrc | [highlightjs-mirc](https://github.com/highlightjs/highlightjs-mirc) | -| Mizar | mizar | | -| Mojolicious | mojolicious | | -| Monkey | monkey | | -| Moonscript | moonscript, moon | | -| N1QL | n1ql | | -| NSIS | nsis | | -| Nginx | nginx, nginxconf | | -| Nimrod | nimrod, nim | | -| Nix | nix | | -| OCaml | ocaml, ml | | -| Objective C | objectivec, mm, objc, obj-c | | -| OpenGL Shading Language | glsl | | -| OpenSCAD | openscad, scad | | -| Oracle Rules Language | ruleslanguage | | -| Oxygene | oxygene | | -| PF | pf, pf.conf | | -| PHP | php, php3, php4, php5, php6 | | -| Parser3 | parser3 | | -| Perl | perl, pl, pm | | -| Plaintext: no highlight | plaintext | | -| Pony | pony | | -| PostgreSQL & PL/pgSQL | pgsql, postgres, postgresql | | -| PowerShell | powershell, ps, ps1 | | -| Processing | processing | | -| Prolog | prolog | | -| Properties | properties | | -| Protocol Buffers | protobuf | | -| Puppet | puppet, pp | | -| Python | python, py, gyp | | -| Python profiler results | profile | | -| Q | k, kdb | | -| QML | qml | | -| R | r | | -| Razor CSHTML | cshtml, razor, razor-cshtml | [highlightjs-cshtml-razor](https://github.com/highlightjs/highlightjs-cshtml-razor) | -| ReasonML | reasonml, re | | -| RenderMan RIB | rib | | -| RenderMan RSL | rsl | | -| Roboconf | graph, instances | | -| Robot Framework | robot, rf | [highlightjs-robot](https://github.com/highlightjs/highlightjs-robot) | -| RPM spec files | rpm-specfile, rpm, spec, rpm-spec, specfile | [highlightjs-rpm-specfile](https://github.com/highlightjs/highlightjs-rpm-specfile) | -| Ruby | ruby, rb, gemspec, podspec, thor, irb | | -| Rust | rust, rs | | -| SAS | SAS, sas | | -| SCSS | scss | | -| SQL | sql | | -| STEP Part 21 | p21, step, stp | | -| Scala | scala | | -| Scheme | scheme | | -| Scilab | scilab, sci | | -| Shape Expressions | shexc | [highlightjs-shexc](https://github.com/highlightjs/highlightjs-shexc) | -| Shell | shell, console | | -| Smali | smali | | -| Smalltalk | smalltalk, st | | -| Solidity | solidity, sol | [highlightjs-solidity](https://github.com/highlightjs/highlightjs-solidity) | -| Stan | stan | | -| Stata | stata | | -| Structured Text | iecst, scl, stl, structured-text | [highlightjs-structured-text](https://github.com/highlightjs/highlightjs-structured-text) | -| Stylus | stylus, styl | | -| SubUnit | subunit | | -| Supercollider | supercollider, sc | [highlightjs-supercollider](https://github.com/highlightjs/highlightjs-supercollider) | -| Swift | swift | | -| Tcl | tcl, tk | | -| Terraform (HCL) | terraform, tf, hcl | [highlightjs-terraform](https://github.com/highlightjs/highlightjs-terraform) | -| Test Anything Protocol | tap | | -| TeX | tex | | -| Thrift | thrift | | -| TP | tp | | -| Twig | twig, craftcms | | -| TypeScript | typescript, ts | | -| VB.Net | vbnet, vb | | -| VBScript | vbscript, vbs | | -| VHDL | vhdl | | -| Vala | vala | | -| Verilog | verilog, v | | -| Vim Script | vim | | -| x86 Assembly | x86asm | | -| XL | xl, tao | | -| XQuery | xquery, xpath, xq | | -| YAML | yml, yaml | | -| Zephir | zephir, zep | | - -Languages with the specified package name are defined in separate repositories -and not included in `highlight.pack.js`. -
- -Classes can also be prefixed with either `language-` or `lang-`. - -To make arbitrary text look like code, but without highlighting, use the +Classes may also be prefixed with either `language-` or `lang-`. + +```html +
...
+``` + +### Plaintext and Disabling Highlighting + +To style arbitrary text like code, but without any highlighting, use the `plaintext` class: ```html 
...
``` -To disable highlighting altogether use the `nohighlight` class: +To disable highlighting of a tag completely, use the `nohighlight` class: ```html 
...
``` +### Supported Languages + +Highlight.js supports over 180 different languages in the core library. There are also 3rd party +language plugins available for additional languages. You can find the full list of supported languages +in [SUPPORTED_LANGUAGES.md][9]. + ## Custom Initialization When you need a bit more control over the initialization of @@ -281,7 +114,7 @@ In worker.js: ```js onmessage = (event) => { - importScripts('/highlight.pack.js'); + importScripts('/highlight.min.js'); const result = self.hljs.highlightAuto(event.data); postMessage(result.value); }; @@ -289,27 +122,54 @@ onmessage = (event) => { ## Node.js -You can use highlight.js with node to highlight content before sending it to the browser. -Make sure to use the `.value` property to get the formatted html. +You can use highlight.js with node to highlight content before sending it to the browser. +Make sure to use the `.value` property to get the formatted html. For more info about the returned object refer to the api docs https://highlightjs.readthedocs.io/en/latest/api.html ```js -// require the highlight.js library including all languages +// require the highlight.js library, including all languages const hljs = require('./highlight.js'); const highlightedCode = hljs.highlightAuto('Hello World!').value ``` +Or for a smaller footprint... load just the languages you need. + ```js -// require the highlight.js library without languages -const hljs = require("highlight.js/lib/highlight.js"); +const hljs = require("highlight.js/lib/core"); // require only the core library // separately require languages -hljs.registerLanguage('html', require('highlight.js/lib/languages/html')); -hljs.registerLanguage('sql', require('highlight.js/lib/languages/sql')); -// highlight with providing the language -const highlightedCode = hljs.highlight('html', 'Hello World!').value +hljs.registerLanguage('xml', require('highlight.js/lib/languages/xml')); + +const highlightedCode = hljs.highlight('xml', 'Hello World!').value +``` + + +## ES6 Modules + +First, you'll likely install via `npm` or `yarn` -- see [Getting the Library](#getting-the-library) below. + +In your application: + +```js +import hljs from 'highlight.js'; +``` + +The default import imports all languages. Therefore it is likely to be more efficient to import only the library and the languages you need: + +```js +import hljs from 'highlight.js/lib/core'; +import javascript from 'highlight.js/lib/languages/javascript'; +hljs.registerLanguage('javascript', javascript); ``` +To set the syntax highlighting style, if your build tool processes CSS from your JavaScript entry point, you can also import the stylesheet directly as modules: + +```js +import hljs from 'highlight.js/lib/core'; +import 'highlight.js/styles/github.css'; +``` + + ## Getting the Library You can get highlight.js as a hosted, or custom-build, browser script or @@ -318,22 +178,12 @@ both AMD and CommonJS, so if you wish you can use RequireJS or Browserify without having to build from source. The server module also works perfectly fine with Browserify, but there is the option to use a build specific to browsers rather than something meant for a server. -Head over to the [download page][5] for all the options. -**Don't link to GitHub directly.** The library is not supposed to work straight + +**Do not link to GitHub directly.** The library is not supposed to work straight from the source, it requires building. If none of the pre-packaged options work for you refer to the [building documentation][6]. -**The CDN-hosted package doesn't have all the languages.** Otherwise it'd be -too big. If you don't see the language you need in the ["Common" section][5], -it can be added manually: - -```html - -``` - **On Almond.** You need to use the optimizer to give the module a name. For example: @@ -341,41 +191,84 @@ example: r.js -o name=hljs paths.hljs=/path/to/highlight out=highlight.js ``` +### CDN Hosted -### CommonJS +A prebuilt version of highlight.js bundled with many common languages is hosted by the following CDNs: -You can import Highlight.js as a CommonJS-module: +**cdnjs** ([link](https://cdnjs.com/libraries/highlight.js)) -```bash -npm install highlight.js --save +```html + + + + ``` -In your application: +**jsdelivr** ([link](https://www.jsdelivr.com/package/gh/highlightjs/cdn-release)) -```js -import hljs from 'highlight.js'; +```html + + ``` -The default import imports all languages! Therefore it is likely to be more efficient to import only the library and the languages you need: +**Note:** *The CDN-hosted `highlight.min.js` package doesn't bundle every language.* It would be +very large. You can find our list "common" languages that we bundle by default on our [download page][5]. + +### Self Hosting + +The [download page][5] can quickly generate a custom bundle including only the languages you need. + +Alternatively, you can build a browser package from [source](#source): -```js -import hljs from 'highlight.js/lib/highlight'; -import javascript from 'highlight.js/lib/languages/javascript'; -hljs.registerLanguage('javascript', javascript); +``` +node tools/build.js -t browser :common ``` -To set the syntax highlighting style, if your build tool processes CSS from your JavaScript entry point, you can import the stylesheet directly into your CommonJS-module: +See our [building documentation][6] for more information. -```js -import hljs from 'highlight.js/lib/highlight'; -import 'highlight.js/styles/github.css'; +**Note:** Building from source should always result in the smallest size builds. The website download page is optimized for speed, not size. + + +#### Prebuilt CDN assets + +You can also download and self-host the same assets we serve up via our own CDNs. We publish those builds to the [cdn-release](https://github.com/highlightjs/cdn-release) GitHub repository. You can easily pull individual files off the CDN endpoints with `curl`, etc; if say you only needed `highlight.min.js` and a single CSS file. + +There is also an npm package [@highlightjs/cdn-assets](https://www.npmjs.com/package/@highlightjs/cdn-assets) if pulling the assets in via `npm` or `yarn` would be easier for your build process. + + +### NPM / Node.js server module + +Highlight.js can also be used on the server. The package with all supported languages can be installed from NPM or Yarn: + +```bash +npm install highlight.js +# or +yarn add highlight.js +``` + +Alternatively, you can build it from [source](#source): + +```bash +node tools/build.js -t node ``` +See our [building documentation][6] for more information. + + +### Source + +[Current source][10] is always available on GitHub. + + ## License Highlight.js is released under the BSD License. See [LICENSE][7] file for details. + ## Links The official site for the library is at . @@ -383,7 +276,7 @@ The official site for the library is at . Further in-depth documentation for the API and other topics is at . -Authors and contributors are listed in the [AUTHORS.en.txt][8] file. +Authors and contributors are listed in the [AUTHORS.txt][8] file. [1]: http://highlightjs.readthedocs.io/en/latest/api.html#inithighlightingonload [2]: http://highlightjs.readthedocs.io/en/latest/css-classes-reference.html @@ -392,4 +285,6 @@ Authors and contributors are listed in the [AUTHORS.en.txt][8] file. [5]: https://highlightjs.org/download/ [6]: http://highlightjs.readthedocs.io/en/latest/building-testing.html [7]: https://github.com/highlightjs/highlight.js/blob/master/LICENSE -[8]: https://github.com/highlightjs/highlight.js/blob/master/AUTHORS.en.txt +[8]: https://github.com/highlightjs/highlight.js/blob/master/AUTHORS.txt +[9]: https://github.com/highlightjs/highlight.js/blob/master/SUPPORTED_LANGUAGES.md +[10]: https://github.com/highlightjs/ diff --git a/README.ru.md b/README.ru.md index 198ee969cf..a448e217ed 100644 --- a/README.ru.md +++ b/README.ru.md @@ -12,7 +12,7 @@ Highlight.js — это инструмент для подсветки синт ```html - + ``` @@ -85,7 +85,7 @@ addEventListener('load', () => { ```js onmessage = (event) => { - importScripts('/highlight.pack.js'); + importScripts('/highlight.min.js'); const result = self.hljs.highlightAuto(event.data); postMessage(result.value); }; @@ -107,7 +107,8 @@ Highlight.js можно использовать в браузере прямо вручную: ```html - + ``` **Про Almond.** Нужно задать имя модуля в оптимизаторе, например: @@ -130,7 +131,7 @@ Highlight.js распространяется под лицензией BSD. П Более подробная документация по API и другим темам расположена на . -Авторы и контрибьюторы перечислены в файле [AUTHORS.ru.txt][8] file. +Авторы и контрибьюторы перечислены в файле [AUTHORS.txt][8] file. [1]: http://highlightjs.readthedocs.io/en/latest/api.html#inithighlightingonload [2]: http://highlightjs.readthedocs.io/en/latest/css-classes-reference.html @@ -139,4 +140,4 @@ Highlight.js распространяется под лицензией BSD. П [5]: https://highlightjs.org/download/ [6]: http://highlightjs.readthedocs.io/en/latest/building-testing.html [7]: https://github.com/highlightjs/highlight.js/blob/master/LICENSE -[8]: https://github.com/highlightjs/highlight.js/blob/master/AUTHORS.ru.txt +[8]: https://github.com/highlightjs/highlight.js/blob/master/AUTHORS.txt diff --git a/SUPPORTED_LANGUAGES.md b/SUPPORTED_LANGUAGES.md new file mode 100644 index 0000000000..482595f134 --- /dev/null +++ b/SUPPORTED_LANGUAGES.md @@ -0,0 +1,205 @@ +# Supported Languages + +The table below shows the full list of supported languages (and corresponding classes/aliases). Note: Which languages are available may depend on how you've built or included the library in your app. See [Getting the Library][1] in the README. + +Languages that listed a **Package** below are 3rd party languages and are not bundled with the core library. You can find their repositories by following the links. + + +| Language | Classes | Package | +| :-----------------------| :--------------------- | :------ | +| 1C | 1c | | +| 4D | 4d |[highlightjs-4d](https://github.com/highlightjs/highlightjs-4d) | +| ABNF | abnf | | +| Access logs | accesslog | | +| Ada | ada | | +| ARM assembler | armasm, arm | | +| AVR assembler | avrasm | | +| ActionScript | actionscript, as | | +| Alan | alan, i | [highlightjs-alan](https://github.com/highlightjs/highlightjs-alan) | +| AngelScript | angelscript, asc | | +| Apache | apache, apacheconf | | +| AppleScript | applescript, osascript | | +| Arcade | arcade | | +| AsciiDoc | asciidoc, adoc | | +| AspectJ | aspectj | | +| AutoHotkey | autohotkey | | +| AutoIt | autoit | | +| Awk | awk, mawk, nawk, gawk | | +| Axapta | axapta | | +| Bash | bash, sh, zsh | | +| Basic | basic | | +| BNF | bnf | | +| Brainfuck | brainfuck, bf | | +| C# | csharp, cs | | +| C | c, h | | +| C++ | cpp, hpp, cc, hh, c++, h++, cxx, hxx | | +| C/AL | cal | | +| Cache Object Script | cos, cls | | +| CMake | cmake, cmake.in | | +| Coq | coq | | +| CSP | csp | | +| CSS | css | | +| Cap’n Proto | capnproto, capnp | | +| Chaos | chaos, kaos | [highlightjs-chaos](https://github.com/chaos-lang/highlightjs-chaos) | +| Cisco CLI | cisco | [highlightjs-cisco-cli](https://github.com/BMatheas/highlightjs-cisco-cli) | +| Clojure | clojure, clj | | +| CoffeeScript | coffeescript, coffee, cson, iced | | +| CpcdosC+ | cpc | [highlightjs-cpcdos](https://github.com/SPinti-Software/highlightjs-cpcdos) | +| Crmsh | crmsh, crm, pcmk | | +| Crystal | crystal, cr | | +| Cypher (Neo4j) | cypher | [highlightjs-cypher](https://github.com/highlightjs/highlightjs-cypher) | +| D | d | | +| DNS Zone file | dns, zone, bind | | +| DOS | dos, bat, cmd | | +| Dart | dart | | +| Delphi | delphi, dpr, dfm, pas, pascal, freepascal, lazarus, lpr, lfm | | +| Diff | diff, patch | | +| Django | django, jinja | | +| Dockerfile | dockerfile, docker | | +| dsconfig | dsconfig | | +| DTS (Device Tree) | dts | | +| Dust | dust, dst | | +| Dylan | dylan | [highlight-dylan](https://github.com/highlightjs/highlight-dylan) | +| EBNF | ebnf | | +| Elixir | elixir | | +| Elm | elm | | +| Erlang | erlang, erl | | +| Excel | excel, xls, xlsx | | +| Extempore | extempore, xtlang, xtm | [highlightjs-xtlang](https://github.com/highlightjs/highlightjs-xtlang) | +| F# | fsharp, fs | | +| FIX | fix | | +| Fortran | fortran, f90, f95 | | +| G-Code | gcode, nc | | +| Gams | gams, gms | | +| GAUSS | gauss, gss | | +| GDScript | godot, gdscript | [highlightjs-gdscript](https://github.com/highlightjs/highlightjs-gdscript) | +| Gherkin | gherkin | | +| GN for Ninja | gn, gni | [highlightjs-GN](https://github.com/highlightjs/highlightjs-GN/blob/master/gn.js) | +| Go | go, golang | | +| Grammatical Framework | gf | [highlightjs-gf](https://github.com/johnjcamilleri/highlightjs-gf) | +| Golo | golo, gololang | | +| Gradle | gradle | | +| Groovy | groovy | | +| HTML, XML | xml, html, xhtml, rss, atom, xjb, xsd, xsl, plist, svg | | +| HTTP | http, https | | +| Haml | haml | | +| Handlebars | handlebars, hbs, html.hbs, html.handlebars | | +| Haskell | haskell, hs | | +| Haxe | haxe, hx | | +| Hy | hy, hylang | | +| Ini, TOML | ini, toml | | +| Inform7 | inform7, i7 | | +| IRPF90 | irpf90 | | +| JSON | json | | +| Java | java, jsp | | +| JavaScript | javascript, js, jsx | | +| Jolie | jolie, iol, ol | [highlightjs-jolie](https://github.com/xiroV/highlightjs-jolie) | +| Kotlin | kotlin, kt | | +| LaTeX | tex | | +| Leaf | leaf | | +| Lean | lean | [highlightjs-lean](https://github.com/leanprover-community/highlightjs-lean) | +| Lasso | lasso, ls, lassoscript | | +| Less | less | | +| LDIF | ldif | | +| Lisp | lisp | | +| LiveCode Server | livecodeserver | | +| LiveScript | livescript, ls | | +| Lua | lua | | +| Makefile | makefile, mk, mak | | +| Markdown | markdown, md, mkdown, mkd | | +| Mathematica | mathematica, mma, wl | | +| Matlab | matlab | | +| Maxima | maxima | | +| Maya Embedded Language | mel | | +| Mercury | mercury | | +| mIRC Scripting Language | mirc, mrc | [highlightjs-mirc](https://github.com/highlightjs/highlightjs-mirc) | +| Mizar | mizar | | +| Mojolicious | mojolicious | | +| Monkey | monkey | | +| Moonscript | moonscript, moon | | +| N1QL | n1ql | | +| NSIS | nsis | | +| Never | never | [highlightjs-never](https://github.com/never-lang/highlightjs-never) | +| Nginx | nginx, nginxconf | | +| Nim | nimrod | | +| Nix | nix | | +| Object Constraint Language | ocl | [highlightjs-ocl](https://github.com/nhomble/highlightjs-ocl) | +| OCaml | ocaml, ml | | +| Objective C | objectivec, mm, objc, obj-c | | +| OpenGL Shading Language | glsl | | +| OpenSCAD | openscad, scad | | +| Oracle Rules Language | ruleslanguage | | +| Oxygene | oxygene | | +| PF | pf, pf.conf | | +| PHP | php, php3, php4, php5, php6, php7 | | +| Parser3 | parser3 | | +| Perl | perl, pl, pm | | +| Plaintext | plaintext, txt, text | | +| Pony | pony | | +| PostgreSQL & PL/pgSQL | pgsql, postgres, postgresql | | +| PowerShell | powershell, ps, ps1 | | +| Processing | processing | | +| Prolog | prolog | | +| Properties | properties | | +| Protocol Buffers | protobuf | | +| Puppet | puppet, pp | | +| Python | python, py, gyp | | +| Python profiler results | profile | | +| Python REPL | python-repl, pycon | | +| Q | k, kdb | | +| QML | qml | | +| R | r | | +| Razor CSHTML | cshtml, razor, razor-cshtml | [highlightjs-cshtml-razor](https://github.com/highlightjs/highlightjs-cshtml-razor) | +| ReasonML | reasonml, re | | +| RenderMan RIB | rib | | +| RenderMan RSL | rsl | | +| Roboconf | graph, instances | | +| Robot Framework | robot, rf | [highlightjs-robot](https://github.com/highlightjs/highlightjs-robot) | +| RPM spec files | rpm-specfile, rpm, spec, rpm-spec, specfile | [highlightjs-rpm-specfile](https://github.com/highlightjs/highlightjs-rpm-specfile) | +| Ruby | ruby, rb, gemspec, podspec, thor, irb | | +| Rust | rust, rs | | +| SAS | SAS, sas | | +| SCSS | scss | | +| SQL | sql | | +| STEP Part 21 | p21, step, stp | | +| Scala | scala | | +| Scheme | scheme | | +| Scilab | scilab, sci | | +| Shape Expressions | shexc | [highlightjs-shexc](https://github.com/highlightjs/highlightjs-shexc) | +| Shell | shell, console | | +| Smali | smali | | +| Smalltalk | smalltalk, st | | +| Solidity | solidity, sol | [highlightjs-solidity](https://github.com/highlightjs/highlightjs-solidity) | +| Stan | stan, stanfuncs | | +| Stata | stata | | +| Structured Text | iecst, scl, stl, structured-text | [highlightjs-structured-text](https://github.com/highlightjs/highlightjs-structured-text) | +| Stylus | stylus, styl | | +| SubUnit | subunit | | +| Supercollider | supercollider, sc | [highlightjs-supercollider](https://github.com/highlightjs/highlightjs-supercollider) | +| Svelte | svelte | [highlightjs-svelte](https://github.com/AlexxNB/highlightjs-svelte) | +| Swift | swift | | +| Tcl | tcl, tk | | +| Terraform (HCL) | terraform, tf, hcl | [highlightjs-terraform](https://github.com/highlightjs/highlightjs-terraform) | +| Test Anything Protocol | tap | | +| Thrift | thrift | | +| TP | tp | | +| Transact-SQL | tsql | [highlightjs-tsql](https://github.com/highlightjs/highlightjs-tsql) | +| Twig | twig, craftcms | | +| TypeScript | typescript, ts | | +| Unicorn Rails log | unicorn-rails-log | [highlightjs-unicorn-rails-log](https://github.com/sweetppro/highlightjs-unicorn-rails-log) +| VB.Net | vbnet, vb | | +| VBA | vba | [highlightjs-vba](https://github.com/dullin/highlightjs-vba) | +| VBScript | vbscript, vbs | | +| VHDL | vhdl | | +| Vala | vala | | +| Verilog | verilog, v | | +| Vim Script | vim | | +| x86 Assembly | x86asm | | +| XL | xl, tao | | +| XQuery | xquery, xpath, xq | | +| YAML | yml, yaml | | +| Zephir | zephir, zep | | + + + +[1]: https://github.com/highlightjs/highlight.js#getting-the-library diff --git a/VERSION_10_BREAKING_CHANGES.md b/VERSION_10_BREAKING_CHANGES.md new file mode 100644 index 0000000000..f60a6c00d5 --- /dev/null +++ b/VERSION_10_BREAKING_CHANGES.md @@ -0,0 +1,40 @@ +## Version 10 Breaking Changes + +Incompatibilities: + +- chore(parser): compressed version 9.x language definitions no longer supported (rebuild them minified) [Josh Goebel][] +- `nohightlight` and `no-highlight` are the only "ignore me" css classes now (`plain` and `text` no longer count) + (to get the old behavior you can customize `noHighlightRe`) +- a grammar with a top-level `self` reference will now always throw an error + (previously in safe mode this would be silently ignored) +- PHP templates are now detected as `php-template`, not `xml` + +Renamed Language Files: + +- chore(parser): `htmlbars.js` now depends on `handlebars.js` [Josh Goebel][] +- chore(parser): rename `nimrod.js` to `nim.js` [Josh Goebel][] +- chore(parser): rename `cs.js` to `csharp.js` [Josh Goebel][] +- chore(parser): rename `tex.js` to `latex.js` [Josh Goebel][] +- chore(parser): effectively rename `cpp.js` to `c-like.js` [Josh Goebel][] +- chore(parser): create new `c.js` (C), depends on `c-like` now [Josh Goebel][] +- chore(parser): create new `cpp.js` (C), depends on `c-like` now [Josh Goebel][] +- This will allow us to clean up C/C++ in the future without another breaking change + by getting this require change out of the way early. + (https://github.com/highlightjs/highlight.js/issues/2146) + +Legacy Browser Issues: + +- **We're now using ES2015 features in the codebase. Internet Explorer 11 is no longer supported.** +- In general legacy browsers are no longer supported. +- chore(parser): remove `load` listener in favor of only the newer `DOMContentLoaded` [Josh Goebel][] + +Removed styles: + +- chore(styles): darkula.css (use darcula.css instead) [Josh Goebel][] + +[Josh Goebel]: https://github.com/yyyc514 + +--- + +Also see: +https://github.com/highlightjs/highlight.js/issues/2273 diff --git a/VERSION_10_UPGRADE.md b/VERSION_10_UPGRADE.md new file mode 100644 index 0000000000..5bb42cf996 --- /dev/null +++ b/VERSION_10_UPGRADE.md @@ -0,0 +1,58 @@ +# Upgrading to Version 10.0 + +Welcome to version 10.0. This a major release and therefore will contain breaking changes. + +## Breaking Changes + +Our normal minor releases try to never break anything, holding all breaking changes for major releases. +We tried to squeeze in as many as we could this time so that after 10.0 ships we'll be back to quiet sailing for a while before we need to push version 11. That said, we're very conservative about what we consider a breaking change. + +*IE, if there it could possibly break things for anyone, it's typically a breaking change.* The fact is a vast majority of users should upgrade and probably not notice any changes at all. + +See [VERSION_10_BREAKING_CHANGES.md](https://github.com/highlightjs/highlight.js/blob/master/VERSION_10_BREAKING_CHANGES.md) for a comprehensive list of breaking changes, but here is a summary... if you use: + +### Core highlight.js lib on the client (with no extra CDN languages) + +Just keep doing that. + +- If you're using `darkula.css`, you'll need to change that to `darcula.css` +- The minified distributable has changed from `.pack.js` to `.min.js`, update your name when you update your URL. +- If your users have very old browsers, they may no longer be supported (no more IE11, etc.). (We're using ES2015 code now.) +- `nohighlight` or `no-highlight` are the only two CSS classes that will SKIP highlighting completely. `*text*` and `*plain*` no longer will do this. + +### Core highlight.js lib on the client (plus additional CDN languages) + +Quite a few grammars have been renamed. Ex: `nimrod.js` is now `nim.js`. + +- Check the renamed grammars to see if you might need to update your links. +- Be aware that you can't use version 9 CDN JS files anymore, they aren't compatible. +- Plus read the above list of items. + +### highlight.js on the server (via NPM) and only use the public API + +If you're just pulling in the FULL library (`require('./highlight.js')`) just keep doing that. You might not need to change anything. + +- If you're manually loading a smaller set of languages and using `registerLanguage` make sure you check out all the renamed grammars and dependency changes. +- Read the client-side lists above also. + +### highlight.js on the server (via NPM) with a custom integration + +Read the complete breaking changes list carefully. + +- Read the client-side lists above also. + +### highlight.js lib on the client, with source directly from our GitHub repo + +That will no longer work. The source needs to be built to work properly and cannot be used "raw" unless you've also setup your own build pipeline (rollup, etc.). Fetch a static build from the CDN, the [cdn-release repo](https://github.com/highlightjs/cdn-release) or use the new [`highlightjs-dist`]() NPM package. + +### highlight.js source code directly from our GitHub repo with a custom integration + +All bets are off, since we only try to guarantee stability of our NPM and CDN builds and the public API. Read all the breaking changes and perhaps skim the commit history. + +- We're using ES6 modules now. +- We're using an entirely new build system. +- The source will likely become more and more modular during the 10.0 timeline. + +## Enjoy and good luck. + +As always if you have any questions or issues, jump on the [Github Issues](https://github.com/highlightjs/highlight.js/issues). diff --git a/demo/demo.js b/demo/demo.js index 3516cb071b..bb46e7a605 100644 --- a/demo/demo.js +++ b/demo/demo.js @@ -1,6 +1,8 @@ (function() { 'use strict'; + hljs.debugMode(); + var $window = $(window), $languages = $('#languages div'), $linkTitle = $('link[title]'), diff --git a/demo/index.html b/demo/index.html index 14becb4962..6ff356a32c 100644 --- a/demo/index.html +++ b/demo/index.html @@ -36,18 +36,18 @@

Styles

- <% _.each(blobs, function(blob) { %> - <% var categories = blob.fileInfo.Category; %> -
class="<%= categories.join(' ') %>"<% } %>> -

<%- blob.fileInfo.Language %>

- 
<%- blob.result %>
+ <% _.each(languages, function(language) { %> + <% var categories = language.categories; %> +
0) { %>class="<%= categories.join(' ') %>"<% } %>> +

<%- language.prettyName %>

+ 
<%- language.sample %>
<% }); %>
- + diff --git a/docs/api.rst b/docs/api.rst index d8039539d3..8b5ee44540 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -4,8 +4,8 @@ Library API Highlight.js exports a few functions as methods of the ``hljs`` object. -``highlight(name, value, ignore_illegals, continuation)`` ---------------------------------------------------------- +``highlight(languageName, code, ignore_illegals, continuation)`` +---------------------------------------------------------------- Core highlighting function. Accepts a language name, or an alias, and a string with the code to highlight. @@ -14,26 +14,35 @@ forces highlighting to finish even in case of detecting illegal syntax for the language instead of throwing an exception. The ``continuation`` is an optional mode stack representing unfinished parsing. When present, the function will restart parsing from this state instead of -initializing a new one. +initializing a new one. This is used internally for `sublanguage` support. + +Note: `continuation` is NOT intended to support line-by-line highlighting +because there is no requirement that a grammar handle linebreaks in any special +way. It's quite possible for a grammar to have a single mode/regex that matches +MANY lines at once. This is not discouraged and entirely up to the grammar. + Returns an object with the following properties: -* ``language``: language name, same as the one passed into a function, returned for consistency with ``highlightAuto`` -* ``relevance``: integer value +* ``language``: language name, same as the name passed in ``languageName``, returned for consistency with ``highlightAuto`` +* ``relevance``: integer value representing the relevance score * ``value``: HTML string with highlighting markup * ``top``: top of the current mode stack +* ``illegal``: boolean representing whether any illegal matches were found +* ``code``: the original raw code -``highlightAuto(value, languageSubset)`` ----------------------------------------- +``highlightAuto(code, languageSubset)`` +--------------------------------------- Highlighting with language detection. Accepts a string with the code to highlight and an optional array of language names and aliases restricting detection to only those languages. The subset can also be set with ``configure``, but the local parameter overrides the option if set. + Returns an object with the following properties: * ``language``: detected language -* ``relevance``: integer value +* ``relevance``: integer value representing the relevance score * ``value``: HTML string with highlighting markup -* ``second_best``: object with the same structure for second-best heuristically detected language, may be absent +* ``second_best``: object with the same structure for second-best heuristically detected language (may be absent) ``fixMarkup(value)`` @@ -66,6 +75,8 @@ Configures global options: * ``useBR``: a flag to generate ``
`` tags instead of new-line characters in the output, useful when code is marked up using a non-``
`` container.
 * ``classPrefix``: a string prefix added before class names in the generated markup, used for backwards compatibility with stylesheets.
 * ``languages``: an array of language names and aliases restricting auto detection to only these languages.
+* ``languageDetectRe``: a regex to configure how CSS class names map to language (allows class names like say `color-as-php` vs the default of `language-php`, etc.)
+* ``noHighlightRe``: a regex to configure which CSS classes are to be skipped completely.
 
 Accepts an object representing options with the values to updated. Other options don't change
 ::
@@ -74,15 +85,14 @@ Accepts an object representing options with the values to updated. Other options
     tabReplace: '    ', // 4 spaces
     classPrefix: ''     // don't append class prefix
                         // … other options aren't changed
-  })
+  });
   hljs.initHighlighting();
 
 
 ``initHighlighting()``
 ----------------------
 
-Applies highlighting to all ``
..
`` blocks on a page.
-
+Applies highlighting to all ``
...
`` blocks on a page.
 
 
 ``initHighlightingOnLoad()``
@@ -91,24 +101,32 @@ Applies highlighting to all ``
..
`` blocks on a page.
 Attaches highlighting to the page load event.
 
 
-``registerLanguage(name, language)``
+``registerLanguage(languageName, languageDefinition)``
 ------------------------------------
 
 Adds new language to the library under the specified name. Used mostly internally.
 
-* ``name``: a string with the name of the language being registered
-* ``language``: a function that returns an object which represents the
+* ``languageName``: a string with the name of the language being registered
+* ``languageDefinition``: a function that returns an object which represents the
   language definition. The function is passed the ``hljs`` object to be able
   to use common regular expressions defined within it.
 
 
+``registerAliases(alias|aliases, {languageName})``
+--------------------------------------------------
+
+Adds new language alias or aliases to the library for the specified language name defined under ``languageName`` key.
+
+* ``alias|aliases``: a string or array with the name of alias being registered
+* ``languageName``: the language name as specified by ``registerLanguage``.
+
+
 ``listLanguages()``
-----------------------------
+-------------------
 
 Returns the languages names list.
 
 
-
 .. _getLanguage:
 
 
@@ -118,3 +136,26 @@ Returns the languages names list.
 Looks up a language by name or alias.
 
 Returns the language object if found, ``undefined`` otherwise.
+
+
+``requireLanguage(name)``
+-------------------------
+
+Looks up a language by name or alias.
+
+This should be used when one language definition depends on another.
+Using this function (vs ``getLanguage``) will provide better error messaging
+when a required language is missing.
+
+Returns the language object if found, raises a hard error otherwise.
+
+
+``debugMode()``
+---------------
+
+Enables *debug/development* mode.  **This mode purposely makes Highlight.js more fragile!  It should only be used for testing and local development (of languages or the library itself).**  By default "Safe Mode" is used, providing the most reliable experience for production usage.
+
+For example, if a new version suddenly had a serious bug (or breaking change) that affected only a single language:
+
+* **In Safe Mode**: All other languages would continue to highlight just fine. The broken language would appear as a code block, but without any highlighting (as if it were plaintext).
+* **In Debug Mode**: All highlighting would stop when an error was encountered and a JavaScript error would be thrown.
diff --git a/docs/building-testing.rst b/docs/building-testing.rst
index 16292cb84a..72c91ca022 100644
--- a/docs/building-testing.rst
+++ b/docs/building-testing.rst
@@ -1,9 +1,27 @@
-Building and testing
+Building and Testing
 ====================
 
-To actually run highlight.js it is necessary to build it for the environment
-where you're going to run it: a browser, the node.js server, etc.
+To use Highlight.js it is first necessary to build it for the environment
+where you plan to execute it: the browser, the Node.js server, etc.
 
+TLDR
+----
+
+Often when contributing a pull-request it's sufficient to build and test only
+the Node.js build. Our CI process will guarantee the browser build tests are all
+still green if you don't run them during development.
+
+::
+
+    npm run build
+    npm run test
+
+The browser library must be built and tested separately:
+
+::
+
+    npm run build-browser
+    npm run test-browser
 
 Building
 --------
@@ -86,3 +104,73 @@ To generate reference rendering use the Developer tool located at
 drop-down menu, as automatic detection is unlikely to work in this case.
 
 
+Building and Testing with Docker
+--------------------------------
+
+If you don't want to install dependencies on your system, you can use the
+included Dockerfile to build a container that will add the source code
+and then deploy a web server for you to preview it. Specifically, after you
+finish with your changes you can build the container from the root of the repository:
+
+::
+
+  docker build -t highlight-js .
+
+
+And then run the container. You will need to expose port 80 on the host for the
+web interface, and note that we are running it in detached (-d) mode.
+
+::
+
+  docker run -d --name highlight-js --rm -p 80:80 highlight-js
+
+
+If your preference is for another port, you can do that too:
+
+
+::
+
+  docker run -d --name highlight-js --rm -p 80:8080 highlight-js
+
+
+Or you can skip binding a port if your intention is to interactively shell
+into the container to use it as a development environment.
+
+
+::
+
+  docker run -d --name highlight-js --rm highlight-js
+
+
+Whatever you choose, you can use docker ps to ensure that it's running.
+
+::
+
+  $ docker ps
+  CONTAINER ID        IMAGE               COMMAND                  CREATED             STATUS              PORTS                NAMES
+  0e15a7c99adf        highlight-js        "docker-entrypoint.s…"   8 seconds ago       Up 7 seconds        0.0.0.0:80->80/tcp   highlight-js
+
+
+Then, open up to http://127.0.0.1/tools/developer.html to see the developer page
+for preview. When you are done, clean up your container.
+
+::
+
+  docker stop highlight-js
+
+If you want a more advanced testing setup, you can bind the source folder when you
+run the container.
+
+::
+
+  docker run -d --name highlight-js --volume $PWD/src:/var/www/html/src --rm -p 80:80 highlight-js
+
+Then if you want to make changes, you can do so locally (the folder is bound as a volume),
+and execute a command to the container to trigger a rebuild:
+
+::
+
+  docker exec highlight-js node tools/build.js :common
+
+
+And then reload the page to see changes. When you finish, don't forget to remove the container.
diff --git a/docs/conf.py b/docs/conf.py
index 5f37c8a794..ea7e2368de 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -41,16 +41,16 @@
 
 # General information about the project.
 project = u'highlight.js'
-copyright = u'2012–2018, Ivan Sagalaev'
+copyright = u'2012–2020, Ivan Sagalaev'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
 # built documents.
 #
 # The short X.Y version.
-version = '9.16'
+version = '10.1'
 # The full version, including alpha/beta/rc tags.
-release = '9.16.0'
+release = '10.1.2'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
diff --git a/docs/css-classes-reference.rst b/docs/css-classes-reference.rst
index c1c958efc4..aa4d45321d 100644
--- a/docs/css-classes-reference.rst
+++ b/docs/css-classes-reference.rst
@@ -136,5 +136,4 @@ Stylable classes
 Language names and aliases
 --------------------------
 
-The language names and aliases table has moved to the project
-[README](https://github.com/highlightjs/highlight.js)
+The language names and aliases table has moved to `SUPPORTED_LANGUAGES.md `_.
diff --git a/docs/index.rst b/docs/index.rst
index 3288758bb5..7ae5b1953f 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -13,9 +13,11 @@ Contents:
 
    api
    language-guide
-   reference
+   mode-reference
    css-classes-reference
    style-guide
+   plugin-api
+   plugin-recipes
    language-contribution
    building-testing
    maintainers-guide
diff --git a/docs/language-contribution.rst b/docs/language-contribution.rst
index 614e816339..f492becb4e 100644
--- a/docs/language-contribution.rst
+++ b/docs/language-contribution.rst
@@ -1,17 +1,25 @@
 Language contributor checklist
 ==============================
 
-1. Put language definition into a .js file
+1. Read ``extra/3RD_PARTY_QUICK_START.md``
 ------------------------------------------
 
+It contains rough high-level steps for creating a 3rd party language grammar for Highlight.js.
+
+
+2. Create a language grammar definition file
+--------------------------------------------
+
 The file defines a function accepting a reference to the library and returning a language object.
-The library parameter is useful to access common modes and regexps. You should not immediately call this function,
-this is done during the build process and details differ for different build targets.
+The library parameter (``hljs``) is useful to access common modes and regexps. You should not
+immediately call this function (you only need to export it). Calling it is handled by the build
+process and details differ for different build targets.
 
 ::
 
-  function(hljs) {
+  export default function(hljs) {
     return {
+      name: "Language Name",
       keywords: 'foo bar',
       contains: [ ..., hljs.NUMBER_MODE, ... ]
     }
@@ -20,8 +28,8 @@ this is done during the build process and details differ for different build tar
 The name of the file is used as a short language identifier and should be usable as a class name in HTML and CSS.
 
 
-2. Provide meta data
---------------------
+3. Add language metadata
+----------------------------
 
 At the top of the file there is a specially formatted comment with meta data processed by a build system.
 Meta data format is simply key-value pairs each occupying its own line:
@@ -34,6 +42,7 @@ Meta data format is simply key-value pairs each occupying its own line:
   Author: John Smith 
   Contributors: Mike Johnson <...@...>, Matt Wilson <...@...>
   Description: Some cool language definition
+  Website: https://superlanguage.example.com
   */
 
 ``Language`` — the only required header giving a human-readable language name.
@@ -44,10 +53,10 @@ Required files aren't processed in any special way.
 The build system just makes sure that they will be in the final package in
 ``LANGUAGES`` object.
 
-The meaning of the other headers is pretty obvious.
+The meaning of the other headers should be pretty obvious.
 
 
-3. Create a code example
+4. Create a code example
 ------------------------
 
 The code example is used both to test language detection and for the demo page
@@ -57,21 +66,27 @@ Take inspiration from other languages in ``test/detect/`` and read
 :ref:`testing instructions ` for more details.
 
 
-4. Write class reference
-------------------------
+5. Write a class reference if your class uses custom classes
+------------------------------------------------------------
+
+A class reference document should typically be placed at the root of your
+language repository: ``css-class-reference.md``
 
-Class reference lives in the :doc:`CSS classes reference `..
 Describe shortly names of all meaningful modes used in your language definition.
 
+Note: If you use custom classes please be aware that all the build-in themes
+are not going to support your custom classes and you should likely also
+distribute your own custom theme.
+
 
-5. Add yourself to AUTHORS.*.txt and CHANGES.md
------------------------------------------------
+6. Request a repository at the ``highlightjs`` organization
+----------------------------------------------------------
 
-If you're a new contributor add yourself to the authors list.
-Also it will be good to update CHANGES.md.
+*This is optional.*  Of course you are free to host your repository anywhere
+you would like.
 
 
-6. Create a pull request
+7. Create a pull request
 ------------------------
 
-Send your contribution as a pull request on GitHub.
+Submit a PR to add your language to `SUPPORTED_LANGUAGES.md`.
diff --git a/docs/language-guide.rst b/docs/language-guide.rst
index 6e598ab8ca..71aa2754ac 100644
--- a/docs/language-guide.rst
+++ b/docs/language-guide.rst
@@ -64,7 +64,7 @@ and most interesting parsing happens inside tags.
 Keywords
 --------
 
-In the simple case language keywords are defined in a string, separated by space:
+In the simple case language keywords can be defined with a string, separated by space:
 
 ::
 
@@ -72,9 +72,11 @@ In the simple case language keywords are defined in a string, separated by space
     keywords: 'else for if while'
   }
 
-Some languages have different kinds of "keywords" that might not be called as such by the language spec
-but are very close to them from the point of view of a syntax highlighter. These are all sorts of "literals", "built-ins", "symbols" and such.
-To define such keyword groups the attribute ``keywords`` becomes an object each property of which defines its own group of keywords:
+Some languages have different kinds of "keywords" that might not be called as
+such by the language spec but are very close to them from the point of view of a
+syntax highlighter. These are all sorts of "literals", "built-ins", "symbols"
+and such. To define such keyword groups the attribute ``keywords`` becomes an
+object each property of which defines its own group of keywords:
 
 ::
 
@@ -85,19 +87,25 @@ To define such keyword groups the attribute ``keywords`` becomes an object each
     }
   }
 
-The group name becomes then a class name in a generated markup enabling different styling for different kinds of keywords.
+The group name becomes the class name in the generated markup enabling different
+themeing for different kinds of keywords.
 
-To detect keywords highlight.js breaks the processed chunk of code into separate words — a process called lexing.
-The "word" here is defined by the regexp ``[a-zA-Z][a-zA-Z0-9_]*`` that works for keywords in most languages.
-Different lexing rules can be defined by the ``lexemes`` attribute:
+To detect keywords highlight.js breaks the processed chunk of code into separate
+words — a process called lexing. By default "words" are matched with the regexp
+``\w+``, and that works well for many languages. Different lexing rules can be
+defined by the magic ``$pattern`` attribute:
 
 ::
 
   {
-    lexemes: '-[a-z]+',
-    keywords: '-import -export'
+    keywords: {
+      $pattern: /-[a-z]+/,        // allow keywords to begin with dash
+      keyword: '-import -export'
+    }
   }
 
+Note: The older ``lexemes`` setting has been deprecated in favor of using
+``keywords.$pattern``. They are functionally identical.
 
 Sub-modes
 ---------
@@ -126,6 +134,8 @@ This is commonly used to define nested modes:
     contains: [hljs.QUOTE_STRING_MODE, 'self']
   }
 
+Note: ``self`` may not be used in the root level ``contains`` of a language.  The root level mode is special and may not be self-referential.
+
 
 Comments
 --------
@@ -184,7 +194,7 @@ For such modes ``className`` attribute should be omitted so they won't generate
 Mode attributes
 ---------------
 
-Other useful attributes are defined in the :doc:`mode reference `.
+Other useful attributes are defined in the :doc:`mode reference `.
 
 
 .. _relevance:
@@ -265,13 +275,14 @@ The goal of Highlight.js is to support whatever regex features Javascript itself
 
 Things we support now that we did not always:
 
-* look-ahead matching for `begin` (#2135)
-* look-ahead matching for `illegal` (#2135)
-* back-references within your regex (#1897)
+* look-ahead regex matching for `begin` (#2135)
+* look-ahead regex matching for `end` (#2237)
+* look-ahead regex matching for `illegal` (#2135)
+* back-references within your regex matches (#1897)
+* look-behind matching (when JS supports it) for `begin` (#2135)
 
 Things we currently know are still issues:
 
-* look-ahead matching for `end` matchers
 * look-behind matching (when JS supports it) for `end` matchers
 
 
diff --git a/docs/language-requests.rst b/docs/language-requests.rst
index 4e4c2f0b61..c8f33d9d02 100644
--- a/docs/language-requests.rst
+++ b/docs/language-requests.rst
@@ -5,13 +5,13 @@ This is a general answer to requests for adding new languages that appear from
 time to time in the highlight.js issue tracker and discussion group.
 
     Highlight.js doesn't have a fundamental plan for implementing languages,
-    instead the project works by accepting language definitions from
-    interested contributors. There are also no rules at the moment forbidding
-    any languages from being added to the library, no matter how obscure or
-    weird.
+    instead the project works by encouraging development of 3rd party language
+    grammars from contributors.  We're also happy to host 3rd party language
+    grammars at the ``highlightjs`` GitHub organization - no matter how obscure
+    or weird.
 
-    This means that there's no point in requesting a new language without
-    providing an implementation for it. If you want to see a particular language
+    This means that *there's no point in requesting a new language without
+    providing an implementation for it*. If you want to see a particular language
     included in highlight.js but cannot implement it, the best way to make it
     happen is to get another developer interested in doing so. Here's our
     :doc:`language-guide`.
diff --git a/docs/maintainers-guide.rst b/docs/maintainers-guide.rst
index 21bfc52396..0ac3719e84 100644
--- a/docs/maintainers-guide.rst
+++ b/docs/maintainers-guide.rst
@@ -14,7 +14,13 @@ Commit policy
 Release process
 ---------------
 
-Releases happen on a 6-week schedule. Currently due to a long break the date of the next release is not set.
+Releases (minor) typically happen on a 6-week schedule.
+
+For major/minor releases you'll be releasing from ``master``.  For patch releases you'll be releasing from a stable branch, such as ``9-16-stable``.  This allows ongoing development of new features to continue in isolation (in master) without those changes leaking into patch releases (which should focus only on fixing breaking changes).
+
+The goal being that minor version series always get more stable over time and that patch releases do not add features.
+
+* For patch releases: First switch to the associated stable branch (i.e., ``9-16-stable``)
 
 * Update CHANGES.md with everything interesting since the last update.
 
@@ -25,9 +31,12 @@ Releases happen on a 6-week schedule. Currently due to a long break the date of
   * ``"version"`` attribute in package-lock.json (run `npm install`)
   * Two places in docs/conf.py (``version`` and ``release``)
 
-* Commit the version changes and tag the commit with the plain version number (no "v." or anything like that)
+* Commit the version changes and tag the commit with the version number (``9.16.2``, no "v" prefix or anything like that)
+
+* For major/minor releases: Create a new ``[major]-[minor]-stable`` branch such as ``9-16-stable``
+
+* Push the commit and the tags (``git push && git push --tags``)
 
-* Push the commit and the tags to master (``git push && git push --tags``)
 
 Pushing the tag triggers the update process which can be monitored at http://highlightjs.org/api/release/
 
diff --git a/docs/reference.rst b/docs/mode-reference.rst
similarity index 71%
rename from docs/reference.rst
rename to docs/mode-reference.rst
index 5869f71aff..afec714bb0 100644
--- a/docs/reference.rst
+++ b/docs/mode-reference.rst
@@ -62,6 +62,19 @@ Regular expression starting a mode. For example a single quote for strings or tw
 If absent, ``begin`` defaults to a regexp that matches anything, so the mode starts immediately.
 
 
+on:begin
+^^^^^^^^^^^
+
+**type**: callback (matchData, response)
+
+This callback is triggered the moment a begin match is detected. ``matchData`` includes the typical regex match data; the full match, match groups, etc. The ``response`` object is used to tell the parser how it should handle the match. It can be also used to temporarily store data.
+
+- ``response.data`` - a simple object data store.  Can be used for building more complex rules where the end rule is dependent on the content of begin, etc.
+- ``response.ignoreMatch()`` - pretend as if this match never happened. The mode is not entered. Continues trying subsequent modes in the current mode's ``contains`` list
+
+For an example of usage see ``END_SAME_AS_BEGIN`` in ``modes.js``.
+
+
 end
 ^^^
 
@@ -72,12 +85,26 @@ Regular expression ending a mode. For example a single quote for strings or "$"
 It's often the case that a beginning regular expression defines the entire mode and doesn't need any special ending.
 For example a number can be defined with ``begin: "\\b\\d+"`` which spans all the digits.
 
-If absent, ``end`` defaults to a regexp that matches anything, so the mode ends immediately.
+If absent, ``end`` defaults to a regexp that matches anything, so the mode ends immediately (after possibly
+matching any ``contains`` sub-modes).
 
 Sometimes a mode can end not by itself but implicitly with its containing (parent) mode.
 This is achieved with :ref:`endsWithParent ` attribute.
 
 
+on:end
+^^^^^^^^^^^
+
+**type**: callback (matchData, response)
+
+This callback is triggered the moment an end match is detected. ``matchData`` includes the typical regex match data; the full match, match groups, etc. The ``response`` object is used to tell the parser how it should handle the match. It can also be used to retrieve data stored from a `begin` callback.
+
+- ``response.data`` - a simple object data store.  Can be used for building more complex rules where the end rule is dependent on the content of begin, etc.
+- ``response.ignoreMatch()`` - pretend as if this match never happened. The mode is not entered. Continues trying subsequent modes in the current mode's ``contains`` list
+
+For an example of usage see ``END_SAME_AS_BEGIN`` in ``modes.js``.
+
+
 beginKeywords
 ^^^^^^^^^^^^^^^^
 
@@ -88,21 +115,25 @@ Used instead of ``begin`` for modes starting with keywords to avoid needless rep
 ::
 
   {
-    begin: '\\b(extends|implements) ',
-    keywords: 'extends implements'
+    begin: '\\b(class|interface)\\b',
+    keywords: 'class interface'
   }
 
-… becomes:
+… can often be shortened to:
 
 ::
 
   {
-    beginKeywords: 'extends implements'
+    beginKeywords: 'class interface'
   }
 
 Unlike the :ref:`keywords ` attribute, this one allows only a simple list of space separated keywords.
 If you do need additional features of ``keywords`` or you just need more keywords for this mode you may include ``keywords`` along with ``beginKeywords``.
 
+Note: ``beginKeywords`` also checks for a ``.`` before or after the keywords and will fail to match if one is found.
+This is to avoid false positives for method calls or property accesses.
+
+Ex. ``class A { ... }`` would match while ``A.class == B.class`` would not.
 
 .. _endsWithParent:
 
@@ -177,15 +208,19 @@ tell it to end the function definition after itself:
 
 .. _endSameAsBegin:
 
-endSameAsBegin
-^^^^^^^^^^^^^^
+endSameAsBegin (deprecated as of 10.1)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+**Deprecated:** *This attribute has been deprecated.*  You should instead use the
+``END_SAME_AS_BEGIN`` mode or use the ``on:begin`` and ``on:end`` attributes to
+build more complex paired matchers.
 
 **type**: boolean
 
 Acts as ``end`` matching exactly the same string that was found by the
 corresponding ``begin`` regexp.
 
-For example, in PostgreSQL string constants can uee "dollar quotes",
+For example, in PostgreSQL string constants can use "dollar quotes",
 consisting of a dollar sign, an optional tag of zero or more characters,
 and another dollar sign. String constant must be ended with the same
 construct using the same tag. It is possible to nest dollar-quoted string
@@ -203,16 +238,22 @@ In this case you can't simply specify the same regexp for ``begin`` and
 ``end`` (say, ``"\\$[a-z]\\$"``), but you can use ``begin: "\\$[a-z]\\$"``
 and ``endSameAsBegin: true``.
 
+
 .. _lexemes:
 
-lexemes
-^^^^^^^
+lexemes (now keywords.$pattern)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 **type**: regexp
 
-A regular expression that extracts individual lexemes from language text to find :ref:`keywords ` among them.
-Default value is ``hljs.IDENT_RE`` which works for most languages.
+A regular expression that extracts individual "words" from the code to compare
+against :ref:`keywords `. The default value is ``\w+`` which works for
+many languages.
 
+Note: It's now recommmended that you use ``keywords.$pattern`` instead of
+``lexemes``, as this makes it easier to keep your keyword pattern associated
+with your keywords themselves, particularly if your keyword configuration is a
+constant that you repeat multiple times within different modes of your grammar.
 
 .. _keywords:
 
@@ -223,8 +264,8 @@ keywords
 
 Keyword definition comes in two forms:
 
-* ``'for while if else weird_voodoo|10 ... '`` -- a string of space-separated keywords with an optional relevance over a pipe
-* ``{'keyword': ' ... ', 'literal': ' ... '}`` -- an object whose keys are names of different kinds of keywords and values are keyword definition strings in the first form
+* ``'for while if|0 else weird_voodoo|10 ... '`` -- a string of space-separated keywords with an optional relevance over a pipe
+* ``{keyword: ' ... ', literal: ' ... ', $pattern: /\w+/ }`` -- an object that describes multiple sets of keywords and the pattern used to find them
 
 For detailed explanation see :doc:`Language definition guide `.
 
@@ -299,7 +340,7 @@ each having all the attributes from the main definition augmented or overridden
 
   {
     className: 'string',
-    contains: [hljs.BACKSLASH_ESCAPE],
+    contains: ['self', hljs.BACKSLASH_ESCAPE],
     relevance: 0,
     variants: [
       {begin: /"/, end: /"/},
@@ -307,6 +348,21 @@ each having all the attributes from the main definition augmented or overridden
     ]
   }
 
+Note: ``variants`` has very specific behavior with regards to ``contains: ['self']``.
+Lets consider the example above. While you might think this would allow you to
+embed any type of string (double or single quoted) within any other string, it
+does not allow for this.
+
+The variants are compiled into to two *discrete* modes::
+
+  { className: 'string', begin: /"/, contains: ['self', ... ] }
+  { className: 'string', begin: /'/, contains: ['self', ... ] }
+
+Each mode's ``self`` refers only to the new expanded mode, not the original mode
+with variants (which no longer exists after compiling).
+
+Further info: https://github.com/highlightjs/highlight.js/issues/826
+
 
 .. _subLanguage:
 
diff --git a/docs/plugin-api.rst b/docs/plugin-api.rst
new file mode 100644
index 0000000000..4b9397ddf6
--- /dev/null
+++ b/docs/plugin-api.rst
@@ -0,0 +1,129 @@
+.. highlight:: javascript
+
+Plugins
+=======
+
+Highlight.js supports plugins.
+
+API
+---
+
+You can add a plugin via the ``addPlugin`` API.
+
+::
+
+  // a plugin can be a class
+  addPlugin(new SimplePlugin())
+  addPlugin(new MoreComplexPlugin(options))
+
+  // or simply a keyed object of functions
+  addPlugin({
+    'after:highlightBlock': (args) => {
+      ...
+    }
+  })
+
+Class based plugins
+^^^^^^^^^^^^^^^^^^^
+
+This approach is useful for more complex plugins that need to deal with
+configuration options or managing state.  Highlight.js will instantiate
+a single instance of
+your class and execute it's callbacks as necessary.
+
+::
+
+  class DataLanguagePlugin {
+    constructor(options) {
+      self.prefix = options.dataPrefix;
+    }
+
+    'after:highlightBlock'({block, result}) {
+      // ...
+    }
+  }
+
+  hljs.addPlugin(new DataLanguagePlugin({dataPrefix: "hljs"}))
+
+Function based plugins
+^^^^^^^^^^^^^^^^^^^^^
+
+This approach is best for simpler plugins.
+
+::
+
+    hljs.addPlugin( {
+      'after:highlightBlock': ({block, result}) => {
+        // move the language from the result into the dataset
+        block.dataset.language = result.language }
+    })
+
+Callbacks
+---------
+
+before:highlight({code, language})
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+This callback function is passed a context object with two keys:
+
+code
+  The code to be highlighted.
+
+language
+  The language grammar that should be used for highlighting.
+
+Your plugin may modify either value and those new values will be used as input
+to the highlighting engine.  If you add a ``result`` key to the object that
+result will be returned as the overall result and the internal highlighting code
+will never even be called.
+
+If you're plugin plans to make its own recursive calls to ``highlight`` you'll
+need to manually handle this. Each time ``highlight`` is called your plugin
+callbacks will also be called - making it easy to get into an infinite loop.
+You'll likely need to use a class based plugin and add a guard so that your
+plugin code is only triggered on the initial call to ``highlight`` and not on
+any internal calls your plugin itself is making.
+
+Note: This callback does not fire from highlighting resulting from auto-language detection.
+
+It returns nothing.
+
+
+after:highlight(result)
+^^^^^^^^^^^^^^^^^^^^^^^
+
+This callback function is passed the ``result`` object after highlighting is
+complete. Your plugin may make any changes it desires to the result object
+and that will be the final return value of the initial call to ``highlight``.
+
+Note: This callback does not fire from highlighting resulting from auto-language detection.
+
+It returns nothing.
+
+
+after:highlightBlock({block, result})
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+This callback function is passed an object with two keys:
+
+block
+  The HTML element of the block that's been highlighted
+
+result
+  The result object returned by `highlight` or `highlightAuto`.
+
+It returns nothing.
+
+
+before:highlightBlock({block, language})
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+This callback function is passed an object with two keys:
+
+block
+  The HTML element of the block that will be highlighted
+
+language
+  The language determined from the class attribute (or undefined).
+
+It returns nothing.
diff --git a/docs/plugin-recipes.rst b/docs/plugin-recipes.rst
new file mode 100644
index 0000000000..4d39883eba
--- /dev/null
+++ b/docs/plugin-recipes.rst
@@ -0,0 +1,26 @@
+.. highlight:: javascript
+
+Recipes
+==============
+
+Below is a collection of useful plugin "recipes" that you might find helpful.
+
+
+data-language
+-------------
+
+Let's say you'd like to track the language that was auto-detected via a
+`data attribute `_.
+This might prove useful if you desired to add a dynamic label
+via CSS with ``:before``, etc.
+
+::
+
+    hljs.addPlugin( {
+      afterHighlightBlock: ({block, result}) => {
+        // move the language from the result into the dataset
+        block.dataset.language = result.language }
+    })
+
+
+
diff --git a/docs/style-guide.rst b/docs/style-guide.rst
index 0aa5b4448b..75b032d77c 100644
--- a/docs/style-guide.rst
+++ b/docs/style-guide.rst
@@ -100,7 +100,7 @@ other meta data if necessary. The format is free:
 
   */
 
-If you're a new contributor add yourself to the authors list in AUTHORS.en.txt
+If you're a new contributor add yourself to the authors list in AUTHORS.txt
 Also update CHANGES.md with your contribution.
 
 Send your contribution as a pull request on GitHub.
diff --git a/extra/.keep b/extra/.keep
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/extra/3RD_PARTY_QUICK_START.md b/extra/3RD_PARTY_QUICK_START.md
new file mode 100644
index 0000000000..5a6b73d5f6
--- /dev/null
+++ b/extra/3RD_PARTY_QUICK_START.md
@@ -0,0 +1,70 @@
+*This is a work in progress.  PRs to improve these docs (or the process) would be welcome.*
+
+## Getting Started
+
+So you'd like to create and share you're own language for Highlight.js.  That's awesome.
+
+Take a look at some of the real-life examples first:
+
+- https://github.com/highlightjs/highlightjs-cypher
+- https://github.com/highlightjs/highlightjs-robots-txt
+
+Basically:
+
+- Checkout highlight-js from github...
+- 3rd party languages are placed into the `extra` directory
+
+So if you had a `xzy` language you'd create an `extra/xyz` folder, and that would be your language module. All paths below are relative to that.
+
+- Put your language file in `src/languages/name.js`.
+- Add detect tests in `test/detect/`.
+- Add markup tests in `test/markup/`.
+- Perhaps add a `package.json` for Node.
+- Add a nice `README`.
+- Don't forget to add a `LICENSE`.
+
+
+## Testing
+
+To test (detect and markup tests), just build highlight.js and test it.  Your tests should be automatically run with the suite:
+
+```
+node ./tools/build.js -t node
+npm run test
+```
+
+If you can't get the auto-detect tests passing you should simply turn off auto-detection for your language in it's definition with `disableAutodetect: true`.  Auto-detection is hard.
+
+
+## Packaging
+
+Users will expect your package to include a minified CDN distributable in your `dist` folder. This should allow them to add the module to their website with only a single `
+
+
diff --git a/test/markup/xml/unquoted-attributes.expect.txt b/test/markup/xml/unquoted-attributes.expect.txt
index 621a51f6fc..98d69e8a02 100644
--- a/test/markup/xml/unquoted-attributes.expect.txt
+++ b/test/markup/xml/unquoted-attributes.expect.txt
@@ -1,9 +1,9 @@
-<img src="/pics/foo.jpg">
-<img src='/pics/foo.jpg'>
+<img src="/pics/foo.jpg">
+<img src='/pics/foo.jpg'>
 <img src=/pics/foo.jpg>
 <img src=/pics/>
 <img src=/pics />
-<img alt=''/>
+<img alt=''/>
 <img alt/>
-<img alt=''>
+<img alt=''>
 <img alt>
diff --git a/test/markup/xquery/computed_inbuilt.expect.txt b/test/markup/xquery/computed_inbuilt.expect.txt
index a902c95d5a..a6633178aa 100644
--- a/test/markup/xquery/computed_inbuilt.expect.txt
+++ b/test/markup/xquery/computed_inbuilt.expect.txt
@@ -1,9 +1,9 @@
-xquery version "3.1";
+xquery version "3.1";
 let $root := element {fn:node-name($e)}
    {$e/@*, 2 * fn:data($e)}
 
 for $node in root($root)
 return
-  element root { root ($node)/text(), attribute root {'root'},
-element not-root{attribute type{"root"}, root($root)}
+  element root { root ($node)/text(), attribute root {'root'},
+element not-root{attribute type{"root"}, root($root)}
 }
diff --git a/test/markup/xquery/direct_method.expect.txt b/test/markup/xquery/direct_method.expect.txt
index 34b756fdd1..591d9f5faf 100644
--- a/test/markup/xquery/direct_method.expect.txt
+++ b/test/markup/xquery/direct_method.expect.txt
@@ -1,12 +1,12 @@
-xquery version "3.1";
-let $var := <root n="x1">"rooting" out 1 or 2 root causes</root>
+xquery version "3.1";
+let $var := <root n="x1">"rooting" out 1 or 2 root causes</root>
 return
-  <result name="test">
+  <result name="test">
     disable highlight for a name such as root {
       for $name in $var
       return
         $name as xs:string
       }
     return to unhighlighted order of things.
-    <test type="{$name}">"rooting" out root causes</test>
+    <test type="{$name}">"rooting" out root causes</test>
   </result>
diff --git a/test/markup/xquery/function_body.expect.txt b/test/markup/xquery/function_body.expect.txt
index a8eb56b69b..fa97b46f3f 100644
--- a/test/markup/xquery/function_body.expect.txt
+++ b/test/markup/xquery/function_body.expect.txt
@@ -2,7 +2,7 @@
 for $n in $node
 return
   element div { switch($n)
-    case 'abc' return 'OK'
+    case 'abc' return 'OK'
   default return 2
   }
 };
diff --git a/test/markup/xquery/prolog_declarations.expect.txt b/test/markup/xquery/prolog_declarations.expect.txt
index 08f101c7b3..3aed0b60eb 100644
--- a/test/markup/xquery/prolog_declarations.expect.txt
+++ b/test/markup/xquery/prolog_declarations.expect.txt
@@ -1,18 +1,18 @@
-xquery version "3.1";
+xquery version "3.1";
 (:~
  : @author Duncan Paterson
  : @version 1.0:)
 
-module namespace app="http://none";
+module namespace app="http://none";
 
-import module namespace config="http://config" at "config.xqm"; (: schema :)
+import module namespace config="http://config" at "config.xqm"; (: schema :)
 
 
 declare copy-namespaces no-preserve, inherit;
 (: switch to preserve, no-inherit:)
 
 declare %private variable $app:maxItems := 12;
-declare context item := doc("catalog.xml");
+declare context item := doc("catalog.xml");
 
 declare %templates:wrap-all function app:helloworld($node as node(), $model as map(*), $name as xs:string?) {
     if ($name) then
diff --git a/test/markup/yaml/inline.expect.txt b/test/markup/yaml/inline.expect.txt
new file mode 100644
index 0000000000..d12626f0ae
--- /dev/null
+++ b/test/markup/yaml/inline.expect.txt
@@ -0,0 +1,3 @@
+foo: [bar, bar2, [1, 2], 3]
+foo: {bar: [1, 2], baz: {inside: 3}}
+foo: ba{}r,ba[]z
diff --git a/test/markup/yaml/inline.txt b/test/markup/yaml/inline.txt
new file mode 100644
index 0000000000..8dfa15b2ea
--- /dev/null
+++ b/test/markup/yaml/inline.txt
@@ -0,0 +1,3 @@
+foo: [bar, bar2, [1, 2], 3]
+foo: {bar: [1, 2], baz: {inside: 3}}
+foo: ba{}r,ba[]z
diff --git a/test/markup/yaml/keys.expect.txt b/test/markup/yaml/keys.expect.txt
index e5fb704eee..4d996f5894 100644
--- a/test/markup/yaml/keys.expect.txt
+++ b/test/markup/yaml/keys.expect.txt
@@ -7,11 +7,11 @@
 some key:
   another key: value
 
-"some key":
-  "another key": value
+"some key":
+  "another key": value
 
-'some key':
-  'another key': value
+'some key':
+  'another key': value
 
 some-key:
   another-key: value
diff --git a/test/markup/yaml/numbers.expect.txt b/test/markup/yaml/numbers.expect.txt
index 969ba53024..2454a1fc4c 100644
--- a/test/markup/yaml/numbers.expect.txt
+++ b/test/markup/yaml/numbers.expect.txt
@@ -3,3 +3,8 @@
 hex: 0x999fff
 numkey999: 1234
 exp: -2.3e-5
+canonical: 2001-12-15T02:59:43.1Z
+iso8601: 2001-12-14t21:59:43.10-05:00
+space: 2001-12-14 21:59:43.10 -5
+nozone: 2001-12-15 2:59:43.10
+date: 2002-12-14
diff --git a/test/markup/yaml/numbers.txt b/test/markup/yaml/numbers.txt
index 7b6c10e2b4..7d39a8eab1 100644
--- a/test/markup/yaml/numbers.txt
+++ b/test/markup/yaml/numbers.txt
@@ -3,3 +3,8 @@ not_hex: 999fff
 hex: 0x999fff
 numkey999: 1234
 exp: -2.3e-5
+canonical: 2001-12-15T02:59:43.1Z
+iso8601: 2001-12-14t21:59:43.10-05:00
+space: 2001-12-14 21:59:43.10 -5
+nozone: 2001-12-15 2:59:43.10
+date: 2002-12-14
diff --git a/test/markup/yaml/string.expect.txt b/test/markup/yaml/string.expect.txt
index 4980e5c938..84899f239b 100644
--- a/test/markup/yaml/string.expect.txt
+++ b/test/markup/yaml/string.expect.txt
@@ -1,6 +1,6 @@
 key: value
-key: 'some value'
-key: "some value"
+key: 'some value'
+key: "some value"
 key: |
   multi-string
   value
diff --git a/test/markup/yaml/tag.expect.txt b/test/markup/yaml/tag.expect.txt
index dbc5645dcd..d777a9b7ac 100644
--- a/test/markup/yaml/tag.expect.txt
+++ b/test/markup/yaml/tag.expect.txt
@@ -1,4 +1,12 @@
 key: !!builtintagname test
 key: !localtagname test
-key: "!notatag"
-key: '!!notatageither'
+key: "!notatag"
+key: '!!notatageither'
+key: !!python/dict test
+key: !!python/name:module.name test
+key: !foo2.bar test
+key: !(foo.bar?):tag test
+key: !named!tag test
+
+--- !<tag:clarkevans.com,2002:invoice>
+invoice: 34843
diff --git a/test/markup/yaml/tag.txt b/test/markup/yaml/tag.txt
index 35f361543d..20ee84a731 100644
--- a/test/markup/yaml/tag.txt
+++ b/test/markup/yaml/tag.txt
@@ -2,3 +2,11 @@ key: !!builtintagname test
 key: !localtagname test
 key: "!notatag"
 key: '!!notatageither'
+key: !!python/dict test
+key: !!python/name:module.name test
+key: !foo2.bar test
+key: !(foo.bar?):tag test
+key: !named!tag test
+
+--- !
+invoice: 34843
diff --git a/test/markup/zephir/default.expect.txt b/test/markup/zephir/default.expect.txt
new file mode 100644
index 0000000000..afd8b2e3fc
--- /dev/null
+++ b/test/markup/zephir/default.expect.txt
@@ -0,0 +1,55 @@
+function testBefore(<Test> a, var b = 5, int c = 10)
+{
+    a->method1();
+
+    return b + c;
+}
+
+namespace Test;
+
+use RuntimeException as RE;
+
+/**
+ * Example comment
+ */
+class Test extends CustomClass implements TestInterface
+{
+    const C1 = null;
+
+    // Magic constant: http://php.net/manual/ru/language.constants.predefined.php
+    const className = __CLASS__;
+
+    public function method1()
+    {
+        int a = 1, b = 2;
+        return a + b;
+    }
+
+    // See fn is allowed like shortcut
+    public fn method2() -> <Test>
+    {
+        call_user_func(function() { echo "hello"; });
+
+
+        [1, 2, 3, 4, 5]->walk(
+            function(int! x) {
+                return x * x;
+            }
+        );
+
+        [1, 2, 3, 4, 5]->walk(
+            function(_, int key) { echo key; }
+        );
+
+        array input = [1, 2, 3, 4, 5];
+
+        input->walk(
+            function(_, int key) { echo key; }
+        );
+
+
+        input->map(x => x * x);
+
+        return this;
+    }
+}
diff --git a/test/markup/zephir/default.txt b/test/markup/zephir/default.txt
new file mode 100644
index 0000000000..8142e7f10c
--- /dev/null
+++ b/test/markup/zephir/default.txt
@@ -0,0 +1,55 @@
+function testBefore( a, var b = 5, int c = 10)
+{
+    a->method1();
+
+    return b + c;
+}
+
+namespace Test;
+
+use RuntimeException as RE;
+
+/**
+ * Example comment
+ */
+class Test extends CustomClass implements TestInterface
+{
+    const C1 = null;
+
+    // Magic constant: http://php.net/manual/ru/language.constants.predefined.php
+    const className = __CLASS__;
+
+    public function method1()
+    {
+        int a = 1, b = 2;
+        return a + b;
+    }
+
+    // See fn is allowed like shortcut
+    public fn method2() -> 
+    {
+        call_user_func(function() { echo "hello"; });
+
+
+        [1, 2, 3, 4, 5]->walk(
+            function(int! x) {
+                return x * x;
+            }
+        );
+
+        [1, 2, 3, 4, 5]->walk(
+            function(_, int key) { echo key; }
+        );
+
+        array input = [1, 2, 3, 4, 5];
+
+        input->walk(
+            function(_, int key) { echo key; }
+        );
+
+
+        input->map(x => x * x);
+
+        return this;
+    }
+}
diff --git a/test/parser/index.js b/test/parser/index.js
index 3dc23c1290..1472a03191 100644
--- a/test/parser/index.js
+++ b/test/parser/index.js
@@ -3,4 +3,5 @@
 describe('hljs', function() {
   require('./reuse-endsWithParent');
   require('./should-not-destroyData');
+  require('./look-ahead-end-matchers');
 });
diff --git a/test/parser/look-ahead-end-matchers.js b/test/parser/look-ahead-end-matchers.js
new file mode 100644
index 0000000000..241e1fd0f6
--- /dev/null
+++ b/test/parser/look-ahead-end-matchers.js
@@ -0,0 +1,31 @@
+const hljs = require('../../build');
+
+describe("parser specifics", function () {
+
+  // CONTEXT: https://github.com/highlightjs/highlight.js/pull/2219
+  describe("a grammar with look-ahead end matchers", () => {
+    it("should match successfully", () => {
+      hljs.registerLanguage('test-language', (hljs) => {
+
+        // broken regex from old Fortran ruleset
+        const PATTERN = {
+          className: "pattern",
+          begin: '[A-Z]{3}',
+          // followed by at least one space
+          end: '\\d{3}(?=\\s+)'
+        }
+
+        return {
+          contains: [PATTERN]
+        };
+      });
+
+      hljs.highlight('test-language', 'ABC123 is the secret. XYZ123. End of string: ABC123').value
+        .should.equal(
+        // one full match at beginning, other match begins with XYZ but then never terminates,
+        // so the end of the parsing finally closes the span tag
+        'ABC123 is the secret. XYZ123. End of string: ABC123'
+      )
+    })
+  })
+})
diff --git a/test/parser/should-not-destroyData.js b/test/parser/should-not-destroyData.js
index bce13a1a31..630e6af3b2 100644
--- a/test/parser/should-not-destroyData.js
+++ b/test/parser/should-not-destroyData.js
@@ -5,6 +5,7 @@ describe("bugs", function () {
   // CONTEXT: https://github.com/highlightjs/highlight.js/pull/2219
   describe("a grammar with a mode that makes a 0 width match", () => {
     it("should instead count it as a 1 character match", () => {
+      hljs.safeMode();
       hljs.registerLanguage('test-language', (hljs) => {
 
         // broken regex from old Fortran ruleset
@@ -28,6 +29,12 @@ describe("bugs", function () {
         // Incorrect prior output:
         // 'The number is 23_longint yes.'
       )
+      hljs.debugMode();
+      should(() => {
+        hljs.highlight('test-language', 'The number is 123_longint yes.').value
+       }).throw(Error, {
+         message: "0 width match regex",
+         languageName: "test-language"})
     })
   })
 })
diff --git a/test/special/buildClassName.js b/test/special/buildClassName.js
index 3b06170f09..304071e431 100644
--- a/test/special/buildClassName.js
+++ b/test/special/buildClassName.js
@@ -1,12 +1,10 @@
 'use strict';
 
-const _ = require('lodash');
-
 describe('block class names',  () => {
   before( () => {
     const testHTML = document.querySelectorAll('#build-classname .hljs');
 
-    this.blocks = _.map(testHTML, 'className');
+    this.blocks = [...testHTML].map((x) => x.className);
   });
 
   it('should add language class name to block',  () => {
diff --git a/test/special/customMarkup.js b/test/special/customMarkup.js
index 5dda648c73..fd88e377cc 100644
--- a/test/special/customMarkup.js
+++ b/test/special/customMarkup.js
@@ -1,13 +1,12 @@
 'use strict';
 
-const _       = require('lodash');
 const utility = require('../utility');
 
 describe('custom markup', () => {
   before(() => {
     const testHTML = document.querySelectorAll('#custom-markup .hljs');
 
-    this.blocks = _.map(testHTML, 'innerHTML');
+    this.blocks = [...testHTML].map(x => x.innerHTML);
   });
 
   it('should replace tabs', () => {
diff --git a/test/special/index.js b/test/special/index.js
index c930842080..1b5419321d 100644
--- a/test/special/index.js
+++ b/test/special/index.js
@@ -1,6 +1,5 @@
 'use strict';
 
-const _        = require('lodash');
 const hljs     = require('../../build');
 const { JSDOM } = require('jsdom');
 const { readFile } = require('fs').promises;
@@ -26,7 +25,7 @@ describe('special cases tests', () => {
     hljs.configure({ useBR: true });
 
     let blocks = document.querySelectorAll('.code');
-    _.each(blocks, hljs.highlightBlock);
+    blocks.forEach(hljs.highlightBlock);
   });
 
   require('./explicitLanguage');
diff --git a/test/special/languageAlias.js b/test/special/languageAlias.js
index 8dc9d71fe6..7f80294cc3 100644
--- a/test/special/languageAlias.js
+++ b/test/special/languageAlias.js
@@ -1,13 +1,12 @@
 'use strict';
 
-const _       = require('lodash');
 const utility = require('../utility');
 
 describe('language alias', () => {
   before(() => {
     const testHTML = document.querySelectorAll('#language-alias .hljs');
 
-    this.blocks = _.map(testHTML, 'innerHTML');
+    this.blocks = [...testHTML].map(x => x.innerHTML);
   });
 
   it('should highlight as aliased language', () => {
diff --git a/test/special/noHighlight.js b/test/special/noHighlight.js
index 57301fbe62..3907b37722 100644
--- a/test/special/noHighlight.js
+++ b/test/special/noHighlight.js
@@ -1,12 +1,10 @@
 'use strict';
 
-const _ = require('lodash');
-
 describe('no highlighting', () => {
   before(() => {
     const testHTML = document.querySelectorAll('#no-highlight pre');
 
-    this.blocks   = _.map(testHTML, 'children[0].innerHTML');
+    this.blocks   = [...testHTML].map((x) => x.children[0].innerHTML);
     this.expected = {
       html:   '<div id="contents">\n  ' +
               '<p>Hello, World!\n</div>',
@@ -30,59 +28,32 @@ describe('no highlighting', () => {
     actual.should.equal(expected);
   });
 
-  it('should keep block unchanged (plain)', () => {
-    const expected = this.expected.html,
-          actual   = this.blocks[2];
-
-    actual.should.equal(expected);
-  });
-
-  it('should keep block unchanged (text)', () => {
-    const expected = this.expected.html,
-          actual   = this.blocks[3];
-
-    actual.should.equal(expected);
-  });
-
   it('should skip pre tags without a child code tag', () => {
     const expected = 'Computer output',
-          actual   = this.blocks[4];
+          actual   = this.blocks[2];
 
     actual.should.equal(expected);
   });
 
   it('should keep block unchanged (unsupported language)', () => {
     const expected = this.expected.python,
-          actual   = this.blocks[5];
+          actual   = this.blocks[3];
 
     actual.should.equal(expected);
   });
 
   it('should keep block unchanged (unsupported lang)', () => {
     const expected = this.expected.python,
-          actual   = this.blocks[6];
+          actual   = this.blocks[4];
 
     actual.should.equal(expected);
   });
 
   it('should keep block unchanged (unsupported prefixed language)', () => {
     const expected = this.expected.python,
-          actual   = this.blocks[7];
-
-    actual.should.equal(expected);
-  });
-
-  it('should highlight class names containing text at the start', () => {
-    const expected = this.expected.javascript,
-          actual   = this.blocks[8];
+          actual   = this.blocks[5];
 
     actual.should.equal(expected);
   });
 
-  it('should highlight class names containing text at the end', () => {
-    const expected = this.expected.javascript,
-          actual   = this.blocks[9];
-
-    actual.should.equal(expected);
-  });
 });
diff --git a/test/tools.js b/test/tools.js
deleted file mode 100644
index 9c3ee1ad2a..0000000000
--- a/test/tools.js
+++ /dev/null
@@ -1,39 +0,0 @@
-'use strict';
-
-const utility = require('../tools/utility');
-const path = require('path');
-const { readFile } = require('fs').promises;
-
-describe("minification tools", () => {
-    it("should replace API calls with minified names", () => {
-        let content = "hljs.COMMENT(); hj.NUMBER_MODE == 0; a = hljs.endRe";
-        content.replace(utility.regex.replaces, utility.replaceClassNames).should.equal(
-            "hljs.C(); hj.NM == 0; a = hljs.eR"
-        );
-    });
-
-    it("should replace API calls with minified names and protect declarations", () => {
-        let content = "hj.NUMBER_MODE == 0; hljs.COMMENT = 1; a = hljs.endRe";
-        content.replace(utility.regex.replaces, utility.replaceClassNames).should.equal(
-            "hj.NM == 0; hljs.C = hljs.COMMENT = 1; a = hljs.eR"
-        );
-    });
-
-    it("should NOT protect non-public member declarations", () => {
-        let content = "hljs.endRe = 3;";
-        content.replace(utility.regex.replaces, utility.replaceClassNames).should.equal(
-            "hljs.eR = 3;"
-        );
-    });
-
-    it("should assign API_REPLACES to the REPLACES dictionary in the highlight.js code", (done) => {
-        readFile(path.join(__dirname, "../src/highlight.js"), 'utf-8').then(function(content) {
-            "abc".should.containEql("bc");
-            content.should.not.containEql("var API_REPLACES = " + JSON.stringify(utility.REPLACES));
-            content.replace(utility.regex.apiReplacesFrom, utility.regex.apiReplacesTo)
-                .should
-                .containEql("var API_REPLACES = " + JSON.stringify(utility.REPLACES));
-            done();
-        });
-    });
-});
diff --git a/test/utility.js b/test/utility.js
index c656648227..60318df03f 100644
--- a/test/utility.js
+++ b/test/utility.js
@@ -1,19 +1,16 @@
 'use strict';
 
-const _        = require('lodash');
 const { readFile } = require('fs').promises;
 const path     = require('path');
 
 // Build a path relative to `test/`
 exports.buildPath = function() {
-  const args  = _.slice(arguments, 0),
+  const args  = [...arguments],
         paths = [__dirname].concat(args);
 
   return path.join.apply(this, paths);
 };
 
-exports.numberToString = _.method('toString');
-
 exports.expectedFile = (filename, encoding, actual) => {
   return readFile(filename, encoding)
     .then(expected => actual.trim().should.equal(expected.trim()));
@@ -23,6 +20,6 @@ exports.setupFile = (filename, encoding, that, testHTML) => {
   return readFile(filename, encoding)
     .then(expected => {
       that.expected = expected.trim();
-      that.blocks   = _.map(testHTML, 'innerHTML');
+      that.blocks   = [...testHTML].map(x => x.innerHTML);
     });
 };
diff --git a/tools/all.js b/tools/all.js
deleted file mode 100644
index 6168be0e53..0000000000
--- a/tools/all.js
+++ /dev/null
@@ -1,34 +0,0 @@
-'use strict';
-
-let _       = require('lodash');
-let path    = require('path');
-let cdn     = require('./cdn');
-let node    = require('./node');
-let browser = require('./browser');
-
-function newBuildDirectory(dir, subdir) {
-  const build = path.join(dir.build, subdir);
-
-  return { build: build };
-}
-
-module.exports = function(commander, dir) {
-  let data = {};
-
-  _.each(['cdn', 'node', 'browser'], function(target) {
-    const newDirectory = newBuildDirectory(dir, target),
-          directory    = _.defaults(newDirectory, dir),
-          options      = _.defaults({ target: target }, commander);
-
-    data[target] = {
-      directory: directory,
-      commander: options
-    };
-  });
-
-  return [].concat(
-    cdn(data.cdn.commander, data.cdn.directory),
-    node(data.node.commander, data.node.directory),
-    browser(data.browser.commander, data.browser.directory)
-  );
-};
diff --git a/tools/browser.js b/tools/browser.js
deleted file mode 100644
index 6f7186c373..0000000000
--- a/tools/browser.js
+++ /dev/null
@@ -1,165 +0,0 @@
-'use strict';
-
-let _        = require('lodash');
-let bluebird = require('bluebird');
-let readFile = bluebird.promisify(require('fs').readFile);
-let path     = require('path');
-
-let registry = require('./tasks');
-let utility  = require('./utility');
-
-let directory;
-
-function templateAllFunc(blobs) {
-  const name = path.join('demo', 'index.html');
-
-  blobs = _.compact(blobs);
-
-  return bluebird.join(
-    readFile(name),
-    utility.getStyleNames(),
-    (template, styles) => ({ template, path, blobs, styles })
-  );
-}
-
-function copyDocs() {
-  const input  = path.join(directory.root, 'docs', '*.rst'),
-        output = path.join(directory.build, 'docs');
-
-  return {
-    startLog: { task: ['log', 'Copying documentation.'] },
-    read: { requires: 'startLog', task: ['glob', utility.glob(input)] },
-    writeLog: { requires: 'read', task: ['log', 'Writing documentation.'] },
-    write: { requires: 'writeLog', task: ['dest', output] }
-  };
-}
-
-function generateDemo(filterCB, readArgs) {
-  let styleDir     = path.join('src', 'styles'),
-      staticArgs   = utility.glob(path.join('demo', '*.min.{js,css}')),
-      imageArgs    = utility.glob(path.join(styleDir,  '*.{png,jpg}'),
-                                  'binary'),
-      stylesArgs   = utility.glob(path.join(styleDir,  '*.css')),
-      demoRoot     = path.join(directory.build, 'demo'),
-      templateArgs = { callback: templateAllFunc },
-      destArgs     = {
-        dir: path.join(demoRoot, 'styles'),
-        encoding: 'binary'
-      };
-
-  return {
-    logStart: { task: ['log', 'Generating demo.'] },
-    readLanguages: { requires: 'logStart', task: ['glob', readArgs] },
-    filterSnippets: { requires: 'readLanguages', task: ['filter', filterCB] },
-    readSnippet: { requires: 'filterSnippets', task: 'readSnippet' },
-    template: {
-      requires: 'readSnippet',
-      task: ['templateAll', templateArgs]
-    },
-    write: {
-      requires: 'template',
-      task: ['write', path.join(demoRoot, 'index.html')]
-    },
-    readStatic: { requires: 'logStart', task: ['glob', staticArgs] },
-    writeStatic: { requires: 'readStatic', task: ['dest', demoRoot] },
-    readStyles: { requires: 'logStart', task: ['glob', stylesArgs] },
-    compressStyles: { requires: 'readStyles', task: 'cssminify' },
-    writeStyles: { requires: 'compressStyles', task: ['dest', destArgs] },
-    readImages: { requires: 'logStart', task: ['glob', imageArgs] },
-    writeImages: { requires:'readImages', task: ['dest', destArgs] },
-    readDemoJS: {
-      requires: 'logStart',
-      task: ['read', path.join('demo', 'demo.js')]
-    },
-    minifyDemoJS: { requires: 'readDemoJS', task: 'jsminify' },
-    writeDemoJS: { requires: 'minifyDemoJS', task: ['dest', demoRoot] },
-    readDemoCSS: {
-      requires: 'logStart',
-      task: ['read', path.join('demo', 'style.css')]
-    },
-    minifyDemoCSS: { requires: 'readDemoCSS', task: 'cssminify' },
-    writeDemoCSS: { requires: 'minifyDemoCSS', task: ['dest', demoRoot] }
-  };
-}
-
-module.exports = function(commander, dir) {
-  directory = dir;
-
-  let hljsExt, output, requiresTask, tasks,
-      replace           = utility.replace,
-      regex             = utility.regex,
-      replaceClassNames = utility.replaceClassNames,
-
-      coreFile     = path.join('src', 'highlight.js'),
-      languages    = utility.glob(path.join('src', 'languages', '*.js')),
-      filterCB     = utility.buildFilterCallback(commander.args),
-      replaceArgs  = replace(regex.header, ''),
-      templateArgs =
-        'hljs.registerLanguage(\'<%= name %>\', <%= content %>);\n';
-
-  tasks = {
-    startLog: { task: ['log', 'Building highlight.js pack file.'] },
-    readCore: { requires: 'startLog', task: ['read', coreFile] },
-    read: { requires: 'startLog', task: ['glob', languages] },
-    filter: { requires: 'read', task: ['filter', filterCB] },
-    reorder: { requires: 'filter', task: 'reorderDeps' },
-    replace: { requires: 'reorder', task: ['replace', replaceArgs] },
-    template: { requires: 'replace', task: ['template', templateArgs] },
-    packageFiles: {
-      requires: ['readCore', 'template'],
-      task: 'packageFiles'
-    }
-  };
-  requiresTask = 'packageFiles';
-
-  if(commander.compress || commander.target === 'cdn') {
-    tasks.compresslog = {
-      requires: requiresTask,
-      task: ['log', 'Compressing highlight.js pack file.']
-    };
-
-    tasks.replace2 = {
-      requires: 'compresslog',
-      task: [ 'replaceSkippingStrings'
-            , replace(regex.replaces, replaceClassNames)
-            ]
-    };
-
-    tasks.replace3 = {
-      requires: 'replace2',
-      task: ['replace', replace(regex.classname, '$1.className')]
-    };
-
-    tasks.replace4 = {
-      requires: 'replace3',
-      task: ['replace', replace(regex.apiReplacesFrom, regex.apiReplacesTo)]
-    };
-
-    tasks.minify  = { requires: 'replace4', task: 'jsminify' };
-    requiresTask  = 'minify';
-  }
-
-  tasks.insertLicenseTag = {
-    requires: requiresTask,
-    task: 'insertLicenseTag'
-  };
-
-  tasks.writelog = {
-    requires: 'insertLicenseTag',
-    task: ['log', 'Writing highlight.js pack file.']
-  };
-
-  hljsExt = commander.target === 'cdn' ? 'min' : 'pack';
-  output  = path.join(directory.build, `highlight.${hljsExt}.js`);
-
-  tasks.write = {
-    requires: 'writelog',
-    task: ['write', output]
-  };
-
-  tasks = (commander.target === 'browser')
-        ? [copyDocs(), generateDemo(filterCB, languages), tasks]
-        : [tasks];
-
-  return utility.toQueue(tasks, registry);
-};
diff --git a/tools/build.js b/tools/build.js
index d39955139b..45ae780309 100644
--- a/tools/build.js
+++ b/tools/build.js
@@ -11,21 +11,20 @@
 // * browser
 //
 //   The default target. This will package up the core `highlight.js` along
-//   with all the language definitions into the file `highlight.pack.js` --
-//   which will be compressed without including the option to disable it. It
-//   also builds the documentation for our readthedocs page, mentioned
-//   above, along with a local instance of the demo at:
+//   with all the language definitions into the file `highlight.js`. A
+//   minified version is also created unless `--no-minify` is passed.
+//   It also builds the documentation for our readthedocs page, mentioned
+//   above, along with a local instance of the demo found at:
 //
 //   .
 //
 // * cdn
 //
-//   This will package up the core `highlight.js` along with all the
-//   language definitions into the file `highlight.min.js` and compresses
-//   all languages and styles into separate files. Since the target is for
-//   CDNs -- like cdnjs and jsdelivr -- it doesn't matter if you put the
-//   option to disable compression, this target is always be compressed. Do
-//   keep in mind that we don't keep the build results in the main
+//   This will package up the core `highlight.js` along with any specified
+//   language definitions into the file `highlight.min.js` and also package
+//   _all_ languages and styles into separate files. The intended use is for
+//   CDNs -- like cdnjs and jsdelivr -- so `--no-minify` is ignored.
+//   Do keep in mind that we don't provide the build results in the main
 //   repository; however, there is a separate repository for those that want
 //   the CDN builds without using a third party site or building it
 //   themselves. For those curious, head over to:
@@ -60,30 +59,49 @@
 
 'use strict';
 
-let commander = require('commander');
-let path      = require('path');
-let Queue     = require('gear').Queue;
-let registry  = require('./tasks');
+const commander = require('commander');
+const path      = require('path');
+const { clean } = require("./lib/makestuff")
+const log = (...args) => console.log(...args)
 
-let build, dir = {};
+const TARGETS = ["cdn", "browser", "node"];
+let dir = {};
 
 commander
   .usage('[options] [...]')
-  .option('-n, --no-compress', 'Disable compression')
+  .option('-n, --no-minify', 'Disable minification')
   .option('-t, --target ', 'Build for target ' +
                                  '[all, browser, cdn, node]',
-                                 /^(browser|cdn|node|all)$/i, 'browser')
+                                  'browser')
   .parse(process.argv);
 
 commander.target = commander.target.toLowerCase();
 
-build     = require(`./${commander.target}`);
 dir.root  = path.dirname(__dirname);
-dir.build = path.join(dir.root, 'build');
+dir.buildRoot = path.join(dir.root, 'build');
 
-new Queue({ registry: registry })
-  .clean(dir.build)
-  .log('Starting build.')
-  .series(build(commander, dir))
-  .log('Finished build.')
-  .run();
+async function doTarget(target, buildDir) {
+  const build     = require(`./build_${target}`);
+  process.env.BUILD_DIR = buildDir;
+  await clean(buildDir);
+  await build.build({languages: commander.args, minify: commander.minify});
+};
+
+async function doBuild() {
+  log ("Starting build.");
+  if (commander.target=="all") {
+    await clean(dir.buildRoot);
+    for (let target of TARGETS) {
+      log (`** Building ${target.toUpperCase()}. **`);
+      let buildDir = path.join(dir.buildRoot, target);
+      await doTarget(target, buildDir);
+    }
+  } else if (TARGETS.includes(commander.target)) {
+    doTarget(commander.target, dir.buildRoot);
+  } else {
+    log(`ERROR: I do not know how to build '${commander.target}'`);
+  }
+  log ("Finished build.");
+}
+
+doBuild()
diff --git a/tools/build_browser.js b/tools/build_browser.js
new file mode 100644
index 0000000000..94fa029e40
--- /dev/null
+++ b/tools/build_browser.js
@@ -0,0 +1,154 @@
+const _        = require('lodash');
+const fs = require("fs").promises;
+const glob = require("glob-promise");
+const path = require("path");
+const zlib = require('zlib');
+const Terser = require("terser");
+const child_process = require('child_process');
+const { getLanguages } = require("./lib/language");
+const { filter } = require("./lib/dependencies");
+const config = require("./build_config");
+const { install, install_cleancss, mkdir, renderTemplate } = require("./lib/makestuff");
+const log = (...args) => console.log(...args);
+const { rollupCode } = require("./lib/bundling.js");
+
+function buildHeader(args) {
+  return "/*\n" +
+  `  Highlight.js ${args.version} (${args.git_sha})\n` +
+  `  License: ${args.license}\n` +
+  `  Copyright (c) ${config.copyrightYears}, ${args.author.name}\n*/`;
+}
+
+async function buildBrowser(options) {
+  var languages = await getLanguages()
+  // filter languages for inclusion in the highlight.js bundle
+  languages = filter(languages, options["languages"]);
+
+  await installDocs();
+  await installDemo(languages);
+
+  log("Preparing languages.")
+  await Promise.all(
+    languages.map(async (lang) => {
+      await lang.compile({terser: config.terser});
+      process.stdout.write(".");
+     } )
+  );
+  log("");
+
+  var size = await buildBrowserHighlightJS(languages, {minify: options.minify})
+
+  log("-----")
+  log("Core                :", size.core ,"bytes");
+  if (options.minify)
+    log("Core (min)          :", size.core_min ,"bytes");
+  log("Languages           :",
+    languages.map((el) => el.data.length).reduce((acc, curr) => acc + curr, 0), "bytes");
+  if (options.minify) {
+    log("Languages (min)     :",
+      languages.map((el) => el.minified.length).reduce((acc, curr) => acc + curr, 0), "bytes");
+  }
+  log("highlight.js        :", size.full ,"bytes");
+  if (options.minify) {
+    log("highlight.min.js    :", size.minified ,"bytes");
+    log("highlight.min.js.gz :", zlib.gzipSync(size.minifiedSrc).length ,"bytes");
+  } else {
+    log("highlight.js.gz     :", zlib.gzipSync(size.fullSrc).length ,"bytes");
+  }
+  log("-----");
+}
+
+async function installDemo(languages) {
+  log("Writing demo files.");
+  mkdir("demo");
+  installDemoStyles();
+
+  const assets = await glob("./demo/*.{js,css}");
+  assets.forEach((file) => install(file));
+
+  const css = await glob("styles/*.css", {cwd:"./src"})
+  const styles = css.map((el) => (
+    { "name": _.startCase(path.basename(el,".css")), "path": el }
+  ));
+  renderTemplate("./demo/index.html", "./demo/index.html", { styles , languages });
+}
+
+async function installDocs() {
+  log("Writing docs files.");
+  mkdir("docs");
+
+  let docs = await glob("./docs/*.rst");
+  docs.forEach((file) => install(file));
+}
+
+function installDemoStyles() {
+  log("Writing style files.");
+  mkdir("demo/styles");
+
+  glob.sync("*", {cwd: "./src/styles"}).forEach((file) => {
+    if (file.endsWith(".css"))
+      install_cleancss(`./src/styles/${file}`,`demo/styles/${file}`);
+    else // images, backgrounds, etc
+      install(`./src/styles/${file}`,`demo/styles/${file}`);
+  })
+}
+
+async function buildBrowserHighlightJS(languages, {minify}) {
+  log("Building highlight.js.");
+
+  var git_sha = child_process
+    .execSync("git rev-parse HEAD")
+    .toString().trim()
+    .slice(0,8)
+  var versionDetails = {...require("../package"), git_sha};
+  var header = buildHeader(versionDetails);
+
+  var outFile = `${process.env.BUILD_DIR}/highlight.js`;
+  var minifiedFile = outFile.replace(/js$/,"min.js");
+
+  const input = { ...config.rollup.browser_core.input, input: `src/highlight.js` }
+  const output = { ...config.rollup.browser_core.output, file: outFile };
+  var librarySrc = await rollupCode(input, output);
+
+  // var librarySrc = await fs.readFile("src/highlight.js", {encoding: "utf8"});
+  var coreSize = librarySrc.length;
+
+  // strip off the original top comment
+  librarySrc = librarySrc.replace(/\/\*.*?\*\//s,"");
+
+  var fullSrc = [
+    header, librarySrc,
+    ...languages.map((lang) => lang.module) ].join("\n");
+
+  var tasks = [];
+  tasks.push(fs.writeFile(outFile, fullSrc, {encoding: "utf8"}));
+
+  var core_min = [];
+  var minifiedSrc = "";
+
+  if (minify) {
+    var tersed = Terser.minify(librarySrc, config.terser)
+
+    minifiedSrc = [
+      header, tersed.code,
+      ...languages.map((lang) => lang.minified) ].join("\n");
+
+    // get approximate core minified size
+    core_min = [ header, tersed.code].join().length;
+
+    tasks.push(fs.writeFile(minifiedFile, minifiedSrc, {encoding: "utf8"}));
+  }
+
+  await Promise.all(tasks);
+  return {
+    core: coreSize,
+    core_min: core_min,
+    minified: Buffer.byteLength(minifiedSrc, 'utf8'),
+    minifiedSrc,
+    fullSrc,
+    full: Buffer.byteLength(fullSrc, 'utf8') };
+}
+
+// CDN build uses the exact same highlight.js distributable
+module.exports.buildBrowserHighlightJS = buildBrowserHighlightJS;
+module.exports.build = buildBrowser;
diff --git a/tools/build_cdn.js b/tools/build_cdn.js
new file mode 100644
index 0000000000..0380bc3ed5
--- /dev/null
+++ b/tools/build_cdn.js
@@ -0,0 +1,113 @@
+const fs = require("fs").promises;
+const glob = require("glob");
+const zlib = require('zlib');
+const { getLanguages } = require("./lib/language");
+const { filter } = require("./lib/dependencies");
+const config = require("./build_config");
+const { install, install_cleancss, mkdir } = require("./lib/makestuff");
+const log = (...args) => console.log(...args);
+const { buildBrowserHighlightJS } = require("./build_browser");
+const { buildPackageJSON } = require("./build_node");
+const path = require("path");
+
+async function installPackageJSON() {
+  await buildPackageJSON();
+  let json = require(`${process.env.BUILD_DIR}/package`);
+  json.name = "@highlightjs/cdn-assets";
+  json.description = json.description.concat(" (pre-compiled CDN assets)");
+  fs.writeFile(`${process.env.BUILD_DIR}/package.json`, JSON.stringify(json, null, '   '));
+}
+
+async function buildCDN(options) {
+  install("./LICENSE", "LICENSE");
+  install("./README.CDN.md","README.md");
+  installPackageJSON();
+
+  installStyles();
+
+  // all the languages are built for the CDN and placed into `/languages`
+  const languages = await getLanguages();
+  await installLanguages(languages);
+
+  // filter languages for inclusion in the highlight.js bundle
+  let embedLanguages = filter(languages, options["languages"])
+
+  // it really makes no sense to embed ALL languages with the CDN build, it's
+  // more likely we want to embed NONE and have completely separate run-time
+  // loading of some sort
+  if (embedLanguages.length == languages.length) {
+    embedLanguages = []
+  }
+
+  var size = await buildBrowserHighlightJS(embedLanguages, {minify: options.minify})
+
+  log("-----")
+  log("Embedded Lang       :",
+    embedLanguages.map((el) => el.minified.length).reduce((acc, curr) => acc + curr, 0), "bytes");
+  log("All Lang            :",
+    languages.map((el) => el.minified.length).reduce((acc, curr) => acc + curr, 0), "bytes");
+  log("highlight.js        :",
+    size.full, "bytes");
+
+  if (options.minify) {
+    log("highlight.min.js    :", size.minified ,"bytes");
+    log("highlight.min.js.gz :", zlib.gzipSync(size.minifiedSrc).length ,"bytes");
+  } else {
+    log("highlight.js.gz     :", zlib.gzipSync(size.fullSrc).length ,"bytes");
+  }
+  log("-----");
+}
+
+async function installLanguages(languages) {
+  log("Building language files.");
+  mkdir("languages");
+
+  await Promise.all(
+    languages.map(async (language) => {
+      await buildCDNLanguage(language);
+      process.stdout.write(".");
+     })
+  );
+  log("");
+
+  await Promise.all(
+    languages.filter((l) => l.third_party)
+      .map(async (language) => {
+        await buildDistributable(language);
+     })
+  );
+
+  log("");
+}
+
+function installStyles() {
+  log("Writing style files.");
+  mkdir("styles");
+
+  glob.sync("*", {cwd: "./src/styles"}).forEach((file) => {
+    if (file.endsWith(".css"))
+      install_cleancss(`./src/styles/${file}`,`styles/${file.replace(".css",".min.css")}`);
+    else // images, backgrounds, etc
+      install(`./src/styles/${file}`,`styles/${file}`);
+  })
+}
+
+async function buildDistributable(language) {
+  const filename = `${language.name}.min.js`;
+
+  let distDir = path.join(language.moduleDir,"dist")
+  log(`Building ${distDir}/${filename}.`)
+  await fs.mkdir(distDir, {recursive: true});
+  fs.writeFile(path.join(language.moduleDir,"dist",filename), language.minified);
+
+}
+
+ async function buildCDNLanguage (language) {
+  const filename = `${process.env.BUILD_DIR}/languages/${language.name}.min.js`;
+
+  await language.compile({terser: config.terser});
+  fs.writeFile(filename, language.minified);
+}
+
+module.exports.build = buildCDN;
+
diff --git a/tools/build_config.js b/tools/build_config.js
new file mode 100644
index 0000000000..c32efe993c
--- /dev/null
+++ b/tools/build_config.js
@@ -0,0 +1,60 @@
+const cjsPlugin = require('rollup-plugin-commonjs');
+const jsonPlugin = require('rollup-plugin-json');
+
+module.exports = {
+    build_dir: "build",
+    copyrightYears: "2006-2020",
+    clean_css: {},
+    rollup: {
+      node: {
+        output: { format: "cjs", strict: false },
+        input : {
+          plugins: [
+            cjsPlugin(),
+            jsonPlugin(),
+            {
+              transform: (x) => {
+                if (/var module/.exec(x)) {
+                  // remove shim that only breaks things for rollup
+                  return x.replace(/var module\s*=.*$/m,"")
+                }
+              }
+            }
+          ],
+        },
+      },
+      browser_core: {
+        input: {
+          plugins: [jsonPlugin()]
+        },
+        output: {
+          name: "hljs",
+          format: "iife",
+          footer: "if (typeof exports === 'object' && typeof module !== 'undefined') { module.exports = hljs; }",
+          interop: false,
+        }
+      },
+      browser: {
+        input: {
+          plugins: [
+            cjsPlugin(),
+            jsonPlugin()
+          ]
+        },
+        output: {
+          format: "iife",
+          outro: "return module.exports.definer || module.exports;",
+          interop: false,
+        }
+      }
+    },
+    terser: {
+        "compress": {
+          passes: 2,
+          unsafe: true,
+          warnings: true,
+          dead_code: true,
+          toplevel: "funcs"
+        }
+    }
+}
diff --git a/tools/build_node.js b/tools/build_node.js
new file mode 100644
index 0000000000..ed30b5d675
--- /dev/null
+++ b/tools/build_node.js
@@ -0,0 +1,105 @@
+const fs = require("fs").promises;
+const config = require("./build_config");
+const { getLanguages } = require("./lib/language");
+const { install, mkdir } = require("./lib/makestuff");
+const { filter } = require("./lib/dependencies");
+const { rollupWrite } = require("./lib/bundling.js");
+const log = (...args) => console.log(...args);
+
+async function buildNodeIndex(languages) {
+  const header = "var hljs = require('./core');";
+  const footer = "module.exports = hljs;";
+
+  const registration = languages.map((lang) => {
+    let require = `require('./languages/${lang.name}')`;
+    if (lang.loader) {
+      require = require += `.${lang.loader}`;
+    }
+    return `hljs.registerLanguage('${lang.name}', ${require});`;
+  })
+
+  // legacy
+  await fs.writeFile(`${process.env.BUILD_DIR}/lib/highlight.js`,
+    "// This file has been deprecated in favor of core.js\n" +
+    "var hljs = require('./core');\n"
+  )
+
+  const index = `${header}\n\n${registration.join("\n")}\n\n${footer}`;
+  await fs.writeFile(`${process.env.BUILD_DIR}/lib/index.js`, index);
+}
+
+ async function buildNodeLanguage (language) {
+  const input = { ...config.rollup.node.input, input: language.path }
+  const output = { ...config.rollup.node.output,  file: `${process.env.BUILD_DIR}/lib/languages/${language.name}.js` }
+  await rollupWrite(input, output)
+}
+
+async function buildNodeHighlightJS() {
+  const input = { ...config.rollup.node.input, input: `src/highlight.js` }
+  const output = { ...config.rollup.node.output, file: `${process.env.BUILD_DIR}/lib/core.js` }
+  await rollupWrite(input, output)
+}
+
+async function buildPackageJSON() {
+  const CONTRIBUTOR = /^- (.*) <(.*)>$/
+
+  let authors = await fs.readFile("AUTHORS.txt", {encoding: "utf8"})
+  let lines = authors.split(/\r?\n/)
+  let json = require("../package")
+  json.contributors = lines.reduce((acc, line) => {
+    let matches = line.match(CONTRIBUTOR)
+
+    if (matches) {
+      acc.push({
+        name: matches[1],
+        email: matches[2]
+      })
+    }
+    return acc;
+  }, []);
+  await fs.writeFile(`${process.env.BUILD_DIR}/package.json`, JSON.stringify(json, null, '   '));
+}
+
+async function buildLanguages(languages) {
+  log("Writing languages.");
+  await Promise.all(
+    languages.map(async (lang) =>  {
+      await buildNodeLanguage(lang);
+      process.stdout.write(".");
+    })
+  )
+  log("");
+}
+
+async function buildNode(options) {
+  mkdir("lib/languages");
+  mkdir("scss");
+  mkdir("styles");
+  mkdir("types");
+
+  install("./LICENSE", "LICENSE");
+  install("./README.md","README.md");
+  install("./types/index.d.ts","types/index.d.ts");
+
+  log("Writing styles.");
+  const styles = await fs.readdir("./src/styles/");
+  styles.forEach((file) => {
+    install(`./src/styles/${file}`,`styles/${file}`);
+    install(`./src/styles/${file}`,`scss/${file.replace(".css",".scss")}`);
+  })
+  log("Writing package.json.");
+  await buildPackageJSON();
+
+  let languages = await getLanguages()
+  // filter languages for inclusion in the highlight.js bundle
+  languages = filter(languages, options["languages"]);
+
+  await buildNodeIndex(languages);
+  await buildLanguages(languages);
+
+  log("Writing highlight.js");
+  await buildNodeHighlightJS();
+}
+
+module.exports.build = buildNode;
+module.exports.buildPackageJSON = buildPackageJSON;
diff --git a/tools/cdn.js b/tools/cdn.js
deleted file mode 100644
index 3d93c9c11b..0000000000
--- a/tools/cdn.js
+++ /dev/null
@@ -1,88 +0,0 @@
-'use strict';
-
-let path = require('path');
-
-let browserBuild = require('./browser');
-let registry     = require('./tasks');
-let utility      = require('./utility');
-
-let directory;
-
-function moveLanguages() {
-  let input   = path.join(directory.root, 'src', 'languages', '*.js'),
-      output  = path.join(directory.build, 'languages'),
-      regex   = utility.regex,
-      replace = utility.replace,
-
-      replaceArgs = replace(regex.header, ''),
-      template    = 'hljs.registerLanguage(\'<%= name %>\','+
-                    ' <%= content %>);\n';
-
-  return {
-    startLog: { task: ['log', 'Building language files.'] },
-    read: {
-      requires: 'startLog',
-      task: ['glob', utility.glob(input)]
-    },
-    replace: { requires: 'read', task: ['replace', replaceArgs] },
-    template: { requires: 'replace', task: ['template', template] },
-    replace2: {
-      requires: 'template',
-      task: [ 'replaceSkippingStrings'
-            , replace(regex.replaces, utility.replaceClassNames)
-            ]
-    },
-    replace3: {
-      requires: 'replace2',
-      task: ['replace', replace(regex.classname, '$1.className')]
-    },
-    compressLog: {
-      requires: 'replace3',
-      task: ['log', 'Compressing languages files.']
-    },
-    minify: { requires: 'compressLog', task: 'jsminify' },
-    rename: { requires: 'minify', task: ['rename', { extname: '.min.js' }] },
-    writeLog: {
-      requires: 'rename',
-      task: ['log', 'Writing language files.']
-    },
-    write: { requires: 'writeLog', task: ['dest', output] }
-  };
-}
-
-function moveStyles() {
-  const css     = path.join(directory.root, 'src', 'styles', '*.css'),
-        images  = path.join(directory.root, 'src', 'styles', '*.{jpg,png}'),
-        output  = path.join(directory.build, 'styles'),
-        options = { dir: output, encoding: 'binary' };
-
-  return {
-    startLog: { task: ['log', 'Building style files.'] },
-    readCSS: { requires: 'startLog', task: ['glob', utility.glob(css)] },
-    readImages: {
-      requires: 'startLog',
-      task: ['glob', utility.glob(images, 'binary')]
-    },
-    compressLog: {
-      requires: 'readCSS',
-      task: ['log', 'Compressing style files.']
-    },
-    minify: { requires: 'compressLog', task: 'cssminify' },
-    rename: {
-      requires: 'minify',
-      task: ['rename', { extname: '.min.css' }]
-    },
-    writeLog: {
-      requires: ['rename', 'readImages'],
-      task: ['log', 'Writing style files.']
-    },
-    write: { requires: 'writeLog', task: ['dest', options] }
-  };
-}
-
-module.exports = function(commander, dir) {
-  directory = dir;
-
-  return utility.toQueue([moveLanguages(), moveStyles()], registry)
-    .concat(browserBuild(commander, dir));
-};
diff --git a/tools/checkAutoDetect.js b/tools/checkAutoDetect.js
index edefd40be6..2b859c60b2 100644
--- a/tools/checkAutoDetect.js
+++ b/tools/checkAutoDetect.js
@@ -8,8 +8,8 @@ let Table    = require('cli-table');
 let colors   = require('colors/safe');
 
 let resultTable = new Table({
-  head: ['expected', 'actual', 'score', '2nd best', 'score'],
-  colWidths: [20,20,10,20, 10],
+  head: ['expected', 'actual', 'score', '2nd best', 'score','info'],
+  colWidths: [20,20,10,20,10,20],
   style: {
     head: ['grey']
   }
@@ -43,6 +43,17 @@ function testAutoDetection(language, index, languages) {
           actual.second_best.relevance ? actual.second_best.relevance : colors.grey('None')
         ]);
       }
+      // equal relevance is flagged
+      if (actual.relevance == actual.second_best.relevance) {
+        return resultTable.push([
+          expected,
+          actual.language,
+          actual.relevance ? colors.yellow(actual.relevance) : colors.grey('None'),
+          actual.second_best.language,
+          actual.second_best.relevance ? colors.yellow(actual.second_best.relevance) : colors.grey('None'),
+          "Relevance match."
+        ]);
+      }
     });
 }
 
@@ -61,7 +72,7 @@ if (resultTable.length === 0) {
   console.log(colors.green('SUCCESS') + ' - ' + colors.green(languages.length) + ' of ' + colors.gray(languages.length) + ' languages passed auto-highlight check!')
 } else {
   console.log(
-    colors.red('FAILURE') + ' - ' + colors.red(resultTable.length) + ' of ' + colors.gray(languages.length) + ' languages failed auto-highlight check.' +
+    colors.red('ISSUES') + ' - ' + colors.red(resultTable.length) + ' of ' + colors.gray(languages.length) + ' languages have potential issues.' +
     '\n' +
     resultTable.toString());
 }
diff --git a/tools/codeformat.js b/tools/codeformat.js
index e6c5727338..081af2364b 100644
--- a/tools/codeformat.js
+++ b/tools/codeformat.js
@@ -1,6 +1,5 @@
 'use strict';
 
-var _         = require('lodash');
 var bluebird  = require('bluebird');
 var path      = require('path');
 var glob      = bluebird.promisify(require('glob'));
diff --git a/tools/developer.html b/tools/developer.html
index 91d875395f..0c32d11a87 100644
--- a/tools/developer.html
+++ b/tools/developer.html
@@ -7,14 +7,22 @@
   
 
   
 
 
-  
highlight.js developer

-
   

+    
Code

     

       
       

-        
+        
+        
         Language: 
       

     

     

-      
Rendered in ... ms [...]

+      
Rendered in ... ms [...]

       
...

     

     

-      
Markup
+      
Markup

       
...

     

   

 
-  
+