diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index ac8d46e..c2810b4 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -28,21 +28,3 @@ jobs: uses: ibiqlik/action-yamllint@master with: strict: true - Python: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Setup Python - uses: actions/setup-python@v1 - with: - python-version: 3.7.7 - architecture: x64 - - name: Install Flake - run: pip install Flake8 - - name: Run Flake8 - uses: suo/flake8-github-action@releases/v1 - with: - # Note: This needs to be the same as the job name. - checkName: 'Python' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index df23f4b..57aafb2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,8 +1,47 @@ --- repos: + - repo: https://github.com/adrienverge/yamllint.git + rev: v1.26.1 + hooks: + - id: yamllint + args: ["-c", ".yamllint", "-s"] + + - repo: https://github.com/psf/black + rev: 20.8b1 + hooks: + - id: black + args: [--line-length=88] + files: '(\.py)$' + language_version: python3 + + # - repo: https://github.com/PyCQA/flake8 + # rev: 3.9.0 + # hooks: + # - id: flake8 + # # Harmonizing flake8 and black + # args: [--max-line-length=88] + # files: '(\.py)$' + + - repo: https://github.com/igorshubovych/markdownlint-cli.git + rev: v0.27.1 + hooks: + - id: markdownlint + + - repo: https://github.com/jumanjihouse/pre-commit-hooks + rev: 2.1.5 + hooks: + - id: forbid-binary + exclude: ^(lib|conf/servers)/.*$ + - id: git-check # Configure in .gitattributes + - id: markdownlint # Configure in .mdlrc + - id: require-ascii + exclude: ^(README.md)$ + - id: script-must-have-extension + - id: shellcheck + - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v3.1.0 + rev: v3.4.0 hooks: - id: check-added-large-files - id: check-ast @@ -16,20 +55,8 @@ repos: - id: check-merge-conflict - id: check-yaml - id: debug-statements - - id: double-quote-string-fixer - id: end-of-file-fixer - id: fix-encoding-pragma - id: mixed-line-ending - id: sort-simple-yaml - id: trailing-whitespace - - - repo: https://github.com/igorshubovych/markdownlint-cli.git - rev: v0.23.1 - hooks: - - id: markdownlint - - - repo: https://github.com/adrienverge/yamllint.git - rev: v1.23.0 - hooks: - - id: yamllint - args: ["-c", ".yamllint", "-s"] diff --git a/README.md b/README.md index cfaf89d..08562ff 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,11 @@ # FX Data Convert 🐳 Action - -[![Release][github-release-image]][github-release-link] +[![Tag][github-tag-image]][github-tag-link] [![Status][gha-image-action-master]][gha-link-action-master] [![Status][gha-image-check-master]][gha-link-check-master] [![Status][gha-image-docker-master]][gha-link-docker-master] [![Status][gha-image-lint-master]][gha-link-lint-master] -[![Telegram Channel][tg-channel-image]][tg-channel-link] -[![Telegram Chat][tg-chat-image]][tg-chat-link] +[![Channel][tg-channel-image]][tg-channel-link] [![Edit][gitpod-image]][gitpod-link] This GitHub Action allows Forex historical data to be converted to different formats. @@ -111,17 +109,14 @@ Foo bar. ## Support - For bugs/features, raise a [new issue at GitHub](https://github.com/FX31337/FX-Data-Convert-Action/issues). -- Join our [Telegram group](https://t.me/FX31337) and [channel](https://t.me/FX31337_Announcements) for help. -[github-release-image]: https://img.shields.io/github/release/FX31337/FX-Data-Convert-Action.svg?logo=github -[github-release-link]: https://github.com/FX31337/FX-Data-Convert-Action/releases +[github-tag-image]: https://img.shields.io/github/tag/FX31337/FX-Data-Convert-Action.svg?logo=github +[github-tag-link]: https://github.com/FX31337/FX-Data-Convert-Action/tags -[tg-channel-image]: https://img.shields.io/badge/Telegram-news-0088CC.svg?logo=telegram -[tg-channel-link]: https://t.me/EA31337_News -[tg-chat-image]: https://img.shields.io/badge/Telegram-chat-0088CC.svg?logo=telegram -[tg-chat-link]: https://t.me/EA31337 +[tg-channel-image]: https://img.shields.io/badge/Telegram-join-0088CC.svg?logo=telegram +[tg-channel-link]: https://t.me/EA31337 [gha-link-action-master]: https://github.com/FX31337/FX-Data-Convert-Action/actions?query=workflow%3AAction+branch%3Amaster [gha-image-action-master]: https://github.com/FX31337/FX-Data-Convert-Action/workflows/Action/badge.svg diff --git a/action.yml b/action.yml index 02a8ec2..890c5ad 100644 --- a/action.yml +++ b/action.yml @@ -1,40 +1,40 @@ # action.yml --- -name: 'FX Data Convert Action' -description: 'Converts Forex historical data to different formats (e.g. CSV to HST/FXT/HCC)' -author: 'kenorb' +name: FX Data Convert Action +description: Converts Forex historical data to different formats (e.g. CSV to HST/FXT/HCC) +author: kenorb inputs: CmdArgs: - description: 'Extra script arguments' - default: '-v' + description: Extra script arguments + default: -v InputFile: - description: 'The filename to convert from' + description: The filename to convert from required: true InputFormat: - description: 'The format of the file to convert from (csv, fxt, hst, hst509)' - default: 'csv' + description: The format of the file to convert from (csv, fxt, hst, hst509) + default: csv OutputFormat: - description: 'The format of the file to convert to (csv, fxt, hcc, hst, hst509)' - default: 'fxt' + description: The format of the file to convert to (csv, fxt, hcc, hst, hst509) + default: fxt ModelingMode: - description: 'Mode of modeling price (for FXT)' + description: Mode of modeling price (for FXT) default: 0 Pair: - description: 'Symbol pair code' - default: 'FOOBAR' + description: Symbol pair code + default: FOOBAR LogLevel: - description: 'Specifies level of verbosity (0-3)' + description: Specifies level of verbosity (0-3) default: 1 # outputs: # foo: # id of output -# description: 'Foo' +# description: Foo runs: - using: 'docker' - image: 'Dockerfile' + using: docker + image: Dockerfile args: - - '${{ inputs.CmdArgs }}' + - ${{ inputs.CmdArgs }} branding: - icon: 'arrow-right-circle' - color: 'blue' + icon: arrow-right-circle + color: blue diff --git a/entrypoint.sh b/entrypoint.sh index 79544b3..1cc3d2e 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -14,14 +14,16 @@ set -u verbose=0 # Implements on-exit trap function. -on_exit() { +on_exit() +{ # @param $1 integer (optional) Exit status. If not set, use '$?'. local exit_status=${1:-$?} [ "$verbose" -eq 1 ] && echo "[INFO] $0 finishes with exit code $exit_status." exit "$exit_status" } # Implements on-error trap function. -on_error() { +on_error() +{ # @param $1 integer (optional) Exit status. If not set, use '$?'. local exit_status=${1:-$?} [ "$verbose" -eq 1 ] && echo "[ERROR] $0 finishes with exit code $exit_status!" @@ -29,7 +31,8 @@ on_error() { exit "$exit_status" } # Displays simple stack trace. -get_backtrace() { +get_backtrace() +{ while caller $((n++)); do :; done } @@ -41,61 +44,64 @@ trap on_exit EXIT trap on_error 1 2 3 15 ERR # Defines variables. -verbose="$( ! [ "${INPUT_LOGLEVEL:-0}" -gt 0 ]; echo $? )" +verbose="$( + ! [ "${INPUT_LOGLEVEL:-0}" -gt 0 ] + echo $? +)" args=() if [ -n "$INPUT_INPUTFILE" ]; then - args+=( "$(printf -- "-i%s" "$INPUT_INPUTFILE")" ) + args+=("$(printf -- "-i%s" "$INPUT_INPUTFILE")") fi # Parse level of verbosity. case "$INPUT_LOGLEVEL" in -1) - args+=( "$(printf -- "%s" "-v")" ) - ;; -2) - args+=( "$(printf -- "%s" "-v")" ) - args+=( "$(printf -- "%s" "-D")" ) - ;; -3) - set -x - args+=( "$(printf -- "%s" "-v")" ) - args+=( "$(printf -- "%s" "-D")" ) - ;; -"") - # Pass-through. - ;; -*) - echo "[ERROR] Unknown log level: $INPUT_LOGLEVEL!" - exit 1 - ;; + 1) + args+=("$(printf -- "%s" "-v")") + ;; + 2) + args+=("$(printf -- "%s" "-v")") + args+=("$(printf -- "%s" "-D")") + ;; + 3) + set -x + args+=("$(printf -- "%s" "-v")") + args+=("$(printf -- "%s" "-D")") + ;; + "") + # Pass-through. + ;; + *) + echo "[ERROR] Unknown log level: $INPUT_LOGLEVEL!" + exit 1 + ;; esac # Parse the inputs and run the script. case "$INPUT_INPUTFORMAT" in -csv) - if [ -n "$INPUT_OUTPUTFORMAT" ]; then - args+=( "$(printf -- "-f%s" "$INPUT_OUTPUTFORMAT")" ) - fi - if [ -n "$INPUT_PAIR" ]; then - args+=( "$(printf -- "-p%s" "$INPUT_PAIR")" ) - fi - if [ -n "$INPUT_MODELINGMODE" ]; then - args+=( "$(printf -- "-m%s" "$INPUT_MODELINGMODE")" ) - fi - /opt/src/fx-data-convert-from-csv.py "${args[@]}" "$@" - ;; -fxt | hcc | hst | hst509) - if [ -n "$INPUT_INPUTFORMAT" ]; then - args+=( "$(printf -- "-f%s" "$INPUT_INPUTFORMAT")" ) - fi - /opt/src/fx-data-convert-to-csv.py "${args[@]}" "$@" - ;; -"") - echo "[ERROR] Please specify the input format!" - exit 1 - ;; -*) - echo "[ERROR] Unknown input format: $INPUT_INPUTFORMAT!" - exit 1 - ;; + csv) + if [ -n "$INPUT_OUTPUTFORMAT" ]; then + args+=("$(printf -- "-f%s" "$INPUT_OUTPUTFORMAT")") + fi + if [ -n "$INPUT_PAIR" ]; then + args+=("$(printf -- "-p%s" "$INPUT_PAIR")") + fi + if [ -n "$INPUT_MODELINGMODE" ]; then + args+=("$(printf -- "-m%s" "$INPUT_MODELINGMODE")") + fi + /opt/src/fx-data-convert-from-csv.py "${args[@]}" "$@" + ;; + fxt | hcc | hst | hst509) + if [ -n "$INPUT_INPUTFORMAT" ]; then + args+=("$(printf -- "-f%s" "$INPUT_INPUTFORMAT")") + fi + /opt/src/fx-data-convert-to-csv.py "${args[@]}" "$@" + ;; + "") + echo "[ERROR] Please specify the input format!" + exit 1 + ;; + *) + echo "[ERROR] Unknown input format: $INPUT_INPUTFORMAT!" + exit 1 + ;; esac diff --git a/src/bstruct/bstruct.py b/src/bstruct/bstruct.py index 44eaec9..5c0e42d 100644 --- a/src/bstruct/bstruct.py +++ b/src/bstruct/bstruct.py @@ -3,13 +3,15 @@ import datetime import binascii + def get_fields_size(spec): # Prepend an endianness mark to prevent calcsize to insert padding bytes - fmt_str = '=' + ''.join(x[1] for x in spec) + fmt_str = "=" + "".join(x[1] for x in spec) return struct.calcsize(fmt_str) -class BStruct(): - def __init__(self, buf, offset = 0): + +class BStruct: + def __init__(self, buf, offset=0): for (name, fmt, *rest) in self._fields: field_size = struct.calcsize(fmt) val = struct.unpack_from(self._endianness + fmt, buf, offset) @@ -23,16 +25,16 @@ def __init__(self, buf, offset = 0): offset += field_size def __str__(self): - ret = '' + ret = "" for (name, _, *fmt) in self._fields: val_repr = getattr(self, name) # Pretty print the value using the custom formatter. if len(fmt): - pp ,= fmt + (pp,) = fmt val_repr = pp(self, getattr(self, name)) - ret += '{} = {}\n'.format(name, val_repr) + ret += "{} = {}\n".format(name, val_repr) return ret @@ -40,15 +42,15 @@ def repack(self): blob_size = get_fields_size(self._fields) if blob_size == 0: - return b'' + return b"" offset = 0 - blob = bytearray(b'\x00' * blob_size) + blob = bytearray(b"\x00" * blob_size) for (name, fmt, *_) in self._fields: field_size = struct.calcsize(fmt) v = getattr(self, name) - if fmt[-1] == 's' or len(fmt) == 1: + if fmt[-1] == "s" or len(fmt) == 1: v = [v] struct.pack_into(self._endianness + fmt, blob, offset, *v) @@ -56,30 +58,39 @@ def repack(self): return blob + # # Pretty printers # def pretty_print_time(obj, x): return datetime.datetime.fromtimestamp(x) + def pretty_print_string(obj, x): - return x.decode('utf-8').rstrip('\0') + return x.decode("utf-8").rstrip("\0") + def pretty_print_wstring(obj, x): - return x.decode('utf-16').rstrip('\0') + return x.decode("utf-16").rstrip("\0") + def pretty_print_bstring(obj, x): return binascii.hexlify(x) + def pretty_print_ignore(obj, x): - return '<...>' + return "<...>" + def pretty_print_hex(obj, x): - return '{:08x}'.format(x) + return "{:08x}".format(x) + def pretty_print_compact(obj, x): - if any(x): return x - return '[\\x00] * {}'.format(len(x)) + if any(x): + return x + return "[\\x00] * {}".format(len(x)) + def pretty_print_decimal_p5(obj, x): - return '{0:.5f}'.format(x) + return "{0:.5f}".format(x) diff --git a/src/bstruct/bstruct_defs.py b/src/bstruct/bstruct_defs.py index f975d72..625258a 100644 --- a/src/bstruct/bstruct_defs.py +++ b/src/bstruct/bstruct_defs.py @@ -5,108 +5,154 @@ # Structure definitions # class TicksRaw(BStruct): - _endianness = '<' + _endianness = "<" _fields = [ - ('symbol', '12s', pretty_print_string), - ('time', 'I', pretty_print_time), - ('bid', 'd'), - ('ask', 'd'), - ('counter', 'I'), - ('unknown', 'I'), - ] + ("symbol", "12s", pretty_print_string), + ("time", "I", pretty_print_time), + ("bid", "d"), + ("ask", "d"), + ("counter", "I"), + ("unknown", "I"), + ] _truncate = True _size = get_fields_size(_fields) - assert(_size == 40) + assert _size == 40 + class SymbolSel(BStruct): - _endianness = '<' + _endianness = "<" _fields = [ - ('symbol', '12s', pretty_print_string), - ('digits', 'I'), - ('index', 'I'), - ('unknown_1', 'I'), - ('group', 'I'), - ('unknown_2', 'I'), - ('pointSize', 'd', pretty_print_decimal_p5), - ('spread', 'I'), - ('unknown_3', 'I'), - ('tickType', 'I'), - ('unknown_4', 'H'), - ('unknown_5', 'H'), - ('time', 'I', pretty_print_time), - ('unknown_6', 'I'), - ('bid', 'd'), - ('ask', 'd'), - ('sessionHigh', 'd'), - ('sessionLow', 'd'), - ('unknown_17', '16s', pretty_print_compact), - ('bid_2', 'd'), - ('ask_2', 'd') - ] + ("symbol", "12s", pretty_print_string), + ("digits", "I"), + ("index", "I"), + ("unknown_1", "I"), + ("group", "I"), + ("unknown_2", "I"), + ("pointSize", "d", pretty_print_decimal_p5), + ("spread", "I"), + ("unknown_3", "I"), + ("tickType", "I"), + ("unknown_4", "H"), + ("unknown_5", "H"), + ("time", "I", pretty_print_time), + ("unknown_6", "I"), + ("bid", "d"), + ("ask", "d"), + ("sessionHigh", "d"), + ("sessionLow", "d"), + ("unknown_17", "16s", pretty_print_compact), + ("bid_2", "d"), + ("ask_2", "d"), + ] _truncate = True _size = get_fields_size(_fields) - assert(_size == 128) + assert _size == 128 + class Symgroups(BStruct): - _endianness = '<' + _endianness = "<" _fields = [ - ('name', '16s', pretty_print_string), - ('description', '60s', pretty_print_string), - ('backgroundColor', 'I') - ] + ("name", "16s", pretty_print_string), + ("description", "60s", pretty_print_string), + ("backgroundColor", "I"), + ] _truncate = True _size = get_fields_size(_fields) - assert(_size == 80) + assert _size == 80 + class SymbolsRaw(BStruct): - _endianness = '<' + _endianness = "<" _fields = [ - ('name', '12s', pretty_print_string), # Symbol name. - ('description', '65s', pretty_print_string), # Symbol description. - ('altName', '11s', pretty_print_string), # Alternative name, e.g. "AUDCAD" (if not equal 'name') - ('baseCurrency', '12s', pretty_print_string), # Base currency. - ('group', 'I'), # Index of group in "symgroups.raw". - ('digits', 'I'), # Number of digits after decimal point for the current symbol prices. - ('tradeMode', 'I'), # Trade mode: 0 = No, 1 = CloseOnly, 2 = Full. - ('backgroundColor', 'I', pretty_print_hex), # Color in "Market Watch" window. - ('id', 'I'), # Unique symbol id. - ('unknown_1', '1508c', pretty_print_ignore), # ???: Some colors (use pretty_print_compact instead to print). - ('unknown_2', 'I'), # ???: E.g. 8, 10, 12, 14, 20, 24, 25, 30, 40, 50, 60, 100, 10000. - ('unknown_3', 'I'), # ???: 1 - Gold, _NQ100, 2 - Currencies, 3 - #IBM, #HPQ, etc. - ('padding_1', '4s', pretty_print_compact), # Padding space - add 4 bytes to align to the next double. - ('unknown_4', 'd'), # ???: E.g. 0.0, 0.1, 0.01, 0.005. - ('unknown_5', '12s', pretty_print_compact), # ???: Empty. - ('spread', 'I'), # Spread in points, 0 for current online spread (variable). - ('unknown_6', 'I'), # ???: Always 0. - ('unknown_7', 'I'), # ???: Always 1. - ('unknown_8', 'I'), # ???: Always 1. - ('profitCalcMode', 'I'), # ???: Profit calculation mode. 0 - Forex; 1 - CFD; 2 - Futures. - ('swapLong', 'd'), # Swap of the buy order. - ('swapShort', 'd'), # Swap of the sell order. - ('3daysSwap', 'I'), # Day of week to charge 3 days swap rollover. - ('padding_2', '4s', pretty_print_compact), # Padding space - add 4 bytes to align to the next double. - ('contractSize', 'd'), # Trade contract size (lot size in units). - ('unknown_9', '16s', pretty_print_compact), # ???: Empty. - ('stopsLevel', 'I'), # Minimal indention in points from the current close price to place Stop orders. - ('unknown_10', 'I'), # ???: Disabled for #HPQ, #IBM, etc. - ('unknown_11', 'I'), # ???: 0 - Currencies, 1 - Futures. - ('padding_3', '4s', pretty_print_compact), # Padding space - add 4 bytes to align to the next double. - ('marginInit', 'd'), # Margin init (0 = contractSize). - ('marginMaintenance', 'd'), # Margin maintenance - ('marginHedged', 'd'), # Margin hedged - ('marginDivider', 'd'), # Leverage calculation: 0...5 - relative to account leverage, > 10 - absolute custom leverage. - ('pointSize', 'd', pretty_print_decimal_p5), # Point size in the quote currency. - ('pointsPerUnit', 'd'), # Points per unit. - ('unknown_12', '24s', pretty_print_compact), # ???: Reserved. - ('marginCurrency', '12s', pretty_print_string), # Margin currency. - ('unknown_13', 'I'), # ???: Always 0. - ('unknown_14', 'I'), # ???: Always 1 for currencies. - ('unknown_15', '96s', pretty_print_compact), # ???: Reserved. - ('unknown_16', 'I'), # ???: E.g. 0, 3, 4, 6, 7, 8, 9, 10, 12, 200. - ] + ("name", "12s", pretty_print_string), # Symbol name. + ("description", "65s", pretty_print_string), # Symbol description. + ( + "altName", + "11s", + pretty_print_string, + ), # Alternative name, e.g. "AUDCAD" (if not equal 'name') + ("baseCurrency", "12s", pretty_print_string), # Base currency. + ("group", "I"), # Index of group in "symgroups.raw". + ( + "digits", + "I", + ), # Number of digits after decimal point for the current symbol prices. + ("tradeMode", "I"), # Trade mode: 0 = No, 1 = CloseOnly, 2 = Full. + ("backgroundColor", "I", pretty_print_hex), # Color in "Market Watch" window. + ("id", "I"), # Unique symbol id. + ( + "unknown_1", + "1508c", + pretty_print_ignore, + ), # ???: Some colors (use pretty_print_compact instead to print). + ( + "unknown_2", + "I", + ), # ???: E.g. 8, 10, 12, 14, 20, 24, 25, 30, 40, 50, 60, 100, 10000. + ( + "unknown_3", + "I", + ), # ???: 1 - Gold, _NQ100, 2 - Currencies, 3 - #IBM, #HPQ, etc. + ( + "padding_1", + "4s", + pretty_print_compact, + ), # Padding space - add 4 bytes to align to the next double. + ("unknown_4", "d"), # ???: E.g. 0.0, 0.1, 0.01, 0.005. + ("unknown_5", "12s", pretty_print_compact), # ???: Empty. + ("spread", "I"), # Spread in points, 0 for current online spread (variable). + ("unknown_6", "I"), # ???: Always 0. + ("unknown_7", "I"), # ???: Always 1. + ("unknown_8", "I"), # ???: Always 1. + ( + "profitCalcMode", + "I", + ), # ???: Profit calculation mode. 0 - Forex; 1 - CFD; 2 - Futures. + ("swapLong", "d"), # Swap of the buy order. + ("swapShort", "d"), # Swap of the sell order. + ("3daysSwap", "I"), # Day of week to charge 3 days swap rollover. + ( + "padding_2", + "4s", + pretty_print_compact, + ), # Padding space - add 4 bytes to align to the next double. + ("contractSize", "d"), # Trade contract size (lot size in units). + ("unknown_9", "16s", pretty_print_compact), # ???: Empty. + ( + "stopsLevel", + "I", + ), # Minimal indention in points from the current close price to place Stop orders. + ("unknown_10", "I"), # ???: Disabled for #HPQ, #IBM, etc. + ("unknown_11", "I"), # ???: 0 - Currencies, 1 - Futures. + ( + "padding_3", + "4s", + pretty_print_compact, + ), # Padding space - add 4 bytes to align to the next double. + ("marginInit", "d"), # Margin init (0 = contractSize). + ("marginMaintenance", "d"), # Margin maintenance + ("marginHedged", "d"), # Margin hedged + ( + "marginDivider", + "d", + ), # Leverage calculation: 0...5 - relative to account leverage, > 10 - absolute custom leverage. + ( + "pointSize", + "d", + pretty_print_decimal_p5, + ), # Point size in the quote currency. + ("pointsPerUnit", "d"), # Points per unit. + ("unknown_12", "24s", pretty_print_compact), # ???: Reserved. + ("marginCurrency", "12s", pretty_print_string), # Margin currency. + ("unknown_13", "I"), # ???: Always 0. + ("unknown_14", "I"), # ???: Always 1 for currencies. + ("unknown_15", "96s", pretty_print_compact), # ???: Reserved. + ("unknown_16", "I"), # ???: E.g. 0, 3, 4, 6, 7, 8, 9, 10, 12, 200. + ] _truncate = True _size = get_fields_size(_fields) - assert(_size == 1936) + assert _size == 1936 + # FXT Header format # @see: https://www.metatrader4.com/en/trading-platform/help/autotrading/tester/tester_fxt @@ -121,228 +167,486 @@ class SymbolsRaw(BStruct): # Source: https://forum.mql4.com/ru/64199/page3 # class FxtHeader(BStruct): - _endianness = '<' + _endianness = "<" _fields = [ # Header layout. - ('headerVersion', 'I'), # Version uint32 0 4 Header version (default: 405). - ('copyright', '64s', pretty_print_string), # Description [64]byte 4 64 Copyright/description (szchar). - ('server', '128s', pretty_print_string), # ServerName [128]byte 68 128 Account server name (szchar). - ('symbol', '12s', pretty_print_string), # Symbol [12]byte 196 12 Symbol (szchar). - ('timeframe', 'i'), # Period uint32 208 4 Timeframe in minutes. - ('modelType', 'i'), # ModelType uint32 212 4 0=EveryTick|1=ControlPoints|2=BarOpen - ('totalBars', 'I'), # ModeledBars uint32 216 4 Number of modeled bars (w/o prolog) - ('firstBarTime', 'I', pretty_print_time), # FirstBarTime uint32 220 4 Bar open time of first tick (w/o prolog) - ('lastBarTime', 'I', pretty_print_time), # LastBarTime uint32 224 4 Bar open time of last tick (w/o prolog) - ('padding1', '4s', pretty_print_ignore), # _ [4]byte 228 4 (alignment to the next double) - + ( + "headerVersion", + "I", + ), # Version uint32 0 4 Header version (default: 405). + ( + "copyright", + "64s", + pretty_print_string, + ), # Description [64]byte 4 64 Copyright/description (szchar). + ( + "server", + "128s", + pretty_print_string, + ), # ServerName [128]byte 68 128 Account server name (szchar). + ( + "symbol", + "12s", + pretty_print_string, + ), # Symbol [12]byte 196 12 Symbol (szchar). + ("timeframe", "i"), # Period uint32 208 4 Timeframe in minutes. + ( + "modelType", + "i", + ), # ModelType uint32 212 4 0=EveryTick|1=ControlPoints|2=BarOpen + ( + "totalBars", + "I", + ), # ModeledBars uint32 216 4 Number of modeled bars (w/o prolog) + ( + "firstBarTime", + "I", + pretty_print_time, + ), # FirstBarTime uint32 220 4 Bar open time of first tick (w/o prolog) + ( + "lastBarTime", + "I", + pretty_print_time, + ), # LastBarTime uint32 224 4 Bar open time of last tick (w/o prolog) + ( + "padding1", + "4s", + pretty_print_ignore, + ), # _ [4]byte 228 4 (alignment to the next double) # Common parameters. - ('modelQuality', 'd'), # ModelQuality float64 232 8 Max. 99.9 - ('baseCurrency', '12s', pretty_print_string), # BaseCurrency [12]byte 240 12 Base currency (szchar) = StringLeft(symbol, 3) - ('spread', 'I'), # Spread uint32 252 4 Spread in points: 0=zero spread = MarketInfo(MODE_SPREAD) - ('digits', 'I'), # Digits uint32 256 4 Symbol digits = MarketInfo(MODE_DIGITS) - ('padding2', '4s', pretty_print_ignore), # _ [4]byte 260 4 (alignment to the next double) - ('pointSize', 'd', pretty_print_decimal_p5), # PointSize float64 264 8 Resolution, ie. 0.0000'1 = MarketInfo(MODE_POINT) - ('minLotSize', 'i'), # MinLotsize uint32 272 4 Min lot size in centi lots (hundredths) = MarketInfo(MODE_MINLOT) * 100 - ('maxLotSize', 'i'), # MaxLotsize uint32 276 4 Max lot size in centi lots (hundredths) = MarketInfo(MODE_MAXLOT) * 100 - ('lotStep', 'i'), # LotStepsize uint32 280 4 Lot stepsize in centi lots (hundredths) = MarketInfo(MODE_LOTSTEP) * 100 - ('stopLevel', 'i'), # StopsLevel uint32 284 4 Orders stop distance in points = MarketInfo(MODE_STOPLEVEL) - ('pendingGTC', 'i'), # PendingsGTC uint32 288 4 Close pending orders at end of day or GTC - ('padding3', '4s', pretty_print_ignore), # _ [4]byte 292 4 (alignment to the next double) - + ("modelQuality", "d"), # ModelQuality float64 232 8 Max. 99.9 + ( + "baseCurrency", + "12s", + pretty_print_string, + ), # BaseCurrency [12]byte 240 12 Base currency (szchar) = StringLeft(symbol, 3) + ( + "spread", + "I", + ), # Spread uint32 252 4 Spread in points: 0=zero spread = MarketInfo(MODE_SPREAD) + ( + "digits", + "I", + ), # Digits uint32 256 4 Symbol digits = MarketInfo(MODE_DIGITS) + ( + "padding2", + "4s", + pretty_print_ignore, + ), # _ [4]byte 260 4 (alignment to the next double) + ( + "pointSize", + "d", + pretty_print_decimal_p5, + ), # PointSize float64 264 8 Resolution, ie. 0.0000'1 = MarketInfo(MODE_POINT) + ( + "minLotSize", + "i", + ), # MinLotsize uint32 272 4 Min lot size in centi lots (hundredths) = MarketInfo(MODE_MINLOT) * 100 + ( + "maxLotSize", + "i", + ), # MaxLotsize uint32 276 4 Max lot size in centi lots (hundredths) = MarketInfo(MODE_MAXLOT) * 100 + ( + "lotStep", + "i", + ), # LotStepsize uint32 280 4 Lot stepsize in centi lots (hundredths) = MarketInfo(MODE_LOTSTEP) * 100 + ( + "stopLevel", + "i", + ), # StopsLevel uint32 284 4 Orders stop distance in points = MarketInfo(MODE_STOPLEVEL) + ( + "pendingGTC", + "i", + ), # PendingsGTC uint32 288 4 Close pending orders at end of day or GTC + ( + "padding3", + "4s", + pretty_print_ignore, + ), # _ [4]byte 292 4 (alignment to the next double) # Profit Calculation parameters. - ('contractSize', 'd'), # ContractSize float64 296 8 ie. 100000 = MarketInfo(MODE_LOTSIZE) - ('tickValue', 'd'), # TickValue float64 304 8 tick value in quote currency (empty) = MarketInfo(MODE_TICKVALUE) - ('tickSize', 'd'), # TickSize float64 312 8 tick size (empty) = MarketInfo(MODE_TICKSIZE) - ('profitCalcMode', 'i'), # ProfitCalculationMode uint32 320 4 0=Forex|1=CFD|2=Futures = MarketInfo(MODE_PROFITCALCMODE) - + ( + "contractSize", + "d", + ), # ContractSize float64 296 8 ie. 100000 = MarketInfo(MODE_LOTSIZE) + ( + "tickValue", + "d", + ), # TickValue float64 304 8 tick value in quote currency (empty) = MarketInfo(MODE_TICKVALUE) + ( + "tickSize", + "d", + ), # TickSize float64 312 8 tick size (empty) = MarketInfo(MODE_TICKSIZE) + ( + "profitCalcMode", + "i", + ), # ProfitCalculationMode uint32 320 4 0=Forex|1=CFD|2=Futures = MarketInfo(MODE_PROFITCALCMODE) # Swap calculation parameters. - ('swapEnabled', 'i'), # SwapEnabled uint32 324 4 if swaps are to be applied - ('swapCalcMode', 'i'), # SwapCalculationMode int32 328 4 0=Points|1=BaseCurrency|2=Interest|3=MarginCurrency = MarketInfo(MODE_SWAPTYPE) - ('padding4', '4s', pretty_print_ignore), # _ [4]byte 332 4 (alignment to the next double) - ('swapLong', 'd'), # SwapLongValue float64 336 8 long overnight swap value = MarketInfo(MODE_SWAPLONG) - ('swapShort', 'd'), # SwapShortValue float64 344 8 short overnight swap values = MarketInfo(MODE_SWAPSHORT) - ('swapRollover', 'i'), # TripleRolloverDay uint32 352 4 weekday of triple swaps = WEDNESDAY (3) - + ( + "swapEnabled", + "i", + ), # SwapEnabled uint32 324 4 if swaps are to be applied + ( + "swapCalcMode", + "i", + ), # SwapCalculationMode int32 328 4 0=Points|1=BaseCurrency|2=Interest|3=MarginCurrency = MarketInfo(MODE_SWAPTYPE) + ( + "padding4", + "4s", + pretty_print_ignore, + ), # _ [4]byte 332 4 (alignment to the next double) + ( + "swapLong", + "d", + ), # SwapLongValue float64 336 8 long overnight swap value = MarketInfo(MODE_SWAPLONG) + ( + "swapShort", + "d", + ), # SwapShortValue float64 344 8 short overnight swap values = MarketInfo(MODE_SWAPSHORT) + ( + "swapRollover", + "i", + ), # TripleRolloverDay uint32 352 4 weekday of triple swaps = WEDNESDAY (3) # Margin calculation parameters. - ('accountLeverage', 'i'), # AccountLeverage uint32 356 4 Account leverage = AccountLeverage(); (default: 100) - ('freeMarginMode', 'i'), # FreeMarginCalculationType uint32 360 4 Free margin calculation type = AccountFreeMarginMode() - ('marginCalcMode', 'i'), # MarginCalculationMode uint32 364 4 Margin calculation mode = MarketInfo(MODE_MARGINCALCMODE) - ('marginStopoutLevel', 'i'), # MarginStopoutLevel uint32 368 4 Margin stopout level = AccountStopoutLevel() - ('marginStopoutMode', 'i'), # MarginStopoutType uint32 372 4 Margin stopout type = AccountStopoutMode() - ('marginRequirements', 'd'), # MarginInit float64 376 8 Initial margin requirement (in units) = MarketInfo(MODE_MARGININIT) - ('marginMaintenanceReq', 'd'), # MarginMaintenance float64 384 8 Maintainance margin requirement (in units) = MarketInfo(MODE_MARGINMAINTENANCE) - ('marginHedgedPosReq', 'd'), # MarginHedged float64 392 8 Hedged margin requirement (in units) = MarketInfo(MODE_MARGINHEDGED) - ('marginLeverageDivider', 'd'), # MarginDivider float64 400 8 Leverage calculation @see example in struct SYMBOL - ('marginCurrency', '12s', pretty_print_string), # MarginCurrency [12]byte 408 12 = AccountCurrency() - ('padding5', '4s', pretty_print_ignore), # _ [4]byte 420 4 (alignment to the next double) - + ( + "accountLeverage", + "i", + ), # AccountLeverage uint32 356 4 Account leverage = AccountLeverage(); (default: 100) + ( + "freeMarginMode", + "i", + ), # FreeMarginCalculationType uint32 360 4 Free margin calculation type = AccountFreeMarginMode() + ( + "marginCalcMode", + "i", + ), # MarginCalculationMode uint32 364 4 Margin calculation mode = MarketInfo(MODE_MARGINCALCMODE) + ( + "marginStopoutLevel", + "i", + ), # MarginStopoutLevel uint32 368 4 Margin stopout level = AccountStopoutLevel() + ( + "marginStopoutMode", + "i", + ), # MarginStopoutType uint32 372 4 Margin stopout type = AccountStopoutMode() + ( + "marginRequirements", + "d", + ), # MarginInit float64 376 8 Initial margin requirement (in units) = MarketInfo(MODE_MARGININIT) + ( + "marginMaintenanceReq", + "d", + ), # MarginMaintenance float64 384 8 Maintainance margin requirement (in units) = MarketInfo(MODE_MARGINMAINTENANCE) + ( + "marginHedgedPosReq", + "d", + ), # MarginHedged float64 392 8 Hedged margin requirement (in units) = MarketInfo(MODE_MARGINHEDGED) + ( + "marginLeverageDivider", + "d", + ), # MarginDivider float64 400 8 Leverage calculation @see example in struct SYMBOL + ( + "marginCurrency", + "12s", + pretty_print_string, + ), # MarginCurrency [12]byte 408 12 = AccountCurrency() + ( + "padding5", + "4s", + pretty_print_ignore, + ), # _ [4]byte 420 4 (alignment to the next double) # Commission calculation parameters. - ('commissionValue', 'd'), # CommissionValue float64 424 8 commission rate - ('commissionCalcMode', 'i'), # CommissionCalculationMode int32 432 4 0=Money|1=Pips|2=Percent @see COMMISSION_MODE_* - ('commissionType', 'i'), # CommissionType int32 436 4 0=RoundTurn|1=PerDeal @see COMMISSION_TYPE_* - + ( + "commissionValue", + "d", + ), # CommissionValue float64 424 8 commission rate + ( + "commissionCalcMode", + "i", + ), # CommissionCalculationMode int32 432 4 0=Money|1=Pips|2=Percent @see COMMISSION_MODE_* + ( + "commissionType", + "i", + ), # CommissionType int32 436 4 0=RoundTurn|1=PerDeal @see COMMISSION_TYPE_* # Later additions. - ('indexOfFirstBar', 'i'), # FirstBar uint32 440 4 Bar number/index??? of first bar (w/o prolog) or 0 for first bar. - ('indexOfLastBar', 'i'), # LastBar uint32 444 4 Bar number/index??? of last bar (w/o prolog) or 0 for last bar. - ('indexOfM1Bar', 'i'), # StartPeriodM1 uint32 448 4 Bar index where modeling started using M1 bars. - ('indexOfM5Bar', 'i'), # StartPeriodM5 uint32 452 4 Bar index where modeling started using M5 bars. - ('indexOfM15Bar', 'i'), # StartPeriodM15 uint32 456 4 Bar index where modeling started using M15 bars. - ('indexOfM30Bar', 'i'), # StartPeriodM30 uint32 460 4 Bar index where modeling started using M30 bars. - ('indexOfH1Bar', 'i'), # StartPeriodH1 uint32 464 4 Bar index where modeling started using H1 bars. - ('indexOfH4Bar', 'i'), # StartPeriodH4 uint32 468 4 Bar index where modeling started using H4 bars. - ('testBeginDate', 'I', pretty_print_time), # TesterSettingFrom uint32 472 4 Begin date from tester settings. - ('testEndDate', 'I', pretty_print_time), # TesterSettingTo uint32 476 4 End date from tester settings. - ('freezeLevel', 'i'), # FreezeDistance uint32 480 4 Order freeze level in points = MarketInfo(MODE_FREEZELEVEL). - ('numberOfErrors', 'I'), # ModelErrors uint32 484 4 Number of errors during model generation (fix errors showing up here before testing). - ('reserved', '240s', pretty_print_ignore), # _ [240]byte 488 240 Unused. - ] + ( + "indexOfFirstBar", + "i", + ), # FirstBar uint32 440 4 Bar number/index??? of first bar (w/o prolog) or 0 for first bar. + ( + "indexOfLastBar", + "i", + ), # LastBar uint32 444 4 Bar number/index??? of last bar (w/o prolog) or 0 for last bar. + ( + "indexOfM1Bar", + "i", + ), # StartPeriodM1 uint32 448 4 Bar index where modeling started using M1 bars. + ( + "indexOfM5Bar", + "i", + ), # StartPeriodM5 uint32 452 4 Bar index where modeling started using M5 bars. + ( + "indexOfM15Bar", + "i", + ), # StartPeriodM15 uint32 456 4 Bar index where modeling started using M15 bars. + ( + "indexOfM30Bar", + "i", + ), # StartPeriodM30 uint32 460 4 Bar index where modeling started using M30 bars. + ( + "indexOfH1Bar", + "i", + ), # StartPeriodH1 uint32 464 4 Bar index where modeling started using H1 bars. + ( + "indexOfH4Bar", + "i", + ), # StartPeriodH4 uint32 468 4 Bar index where modeling started using H4 bars. + ( + "testBeginDate", + "I", + pretty_print_time, + ), # TesterSettingFrom uint32 472 4 Begin date from tester settings. + ( + "testEndDate", + "I", + pretty_print_time, + ), # TesterSettingTo uint32 476 4 End date from tester settings. + ( + "freezeLevel", + "i", + ), # FreezeDistance uint32 480 4 Order freeze level in points = MarketInfo(MODE_FREEZELEVEL). + ( + "numberOfErrors", + "I", + ), # ModelErrors uint32 484 4 Number of errors during model generation (fix errors showing up here before testing). + ( + "reserved", + "240s", + pretty_print_ignore, + ), # _ [240]byte 488 240 Unused. + ] _truncate = False _size = get_fields_size(_fields) - assert(_size == 728) + assert _size == 728 + # FXT Tick data. # The array of modeled bars. # @see: https://www.metatrader4.com/en/trading-platform/help/autotrading/tester/tester_fxt # class FxtTick(BStruct): - _endianness = '<' + _endianness = "<" _fields = [ - ('barTimestamp', 'II', pretty_print_time), # BarTimestamp uint64 0 8 Bar datetime, align with timeframe, unit seconds. - ('open', 'd'), # Open float64 8 8 - ('high', 'd'), # High float64 16 8 - ('low', 'd'), # Low float64 24 8 - ('close', 'd'), # Close float64 32 8 - ('volume', 'II'), # Volume uint64 40 8 Volume (documentation says it's a double, though it's stored as a long int). - ('tickTimestamp', 'i', pretty_print_time), # TickTimestamp uint32 48 4 Tick data timestamp in seconds (the current time within a bar). - ('launchExpert', 'i', pretty_print_time), # LaunchExpert uint32 52 4 Flag to launch an expert (0 - bar will be modified, but the expert will not be launched). - ] + ( + "barTimestamp", + "II", + pretty_print_time, + ), # BarTimestamp uint64 0 8 Bar datetime, align with timeframe, unit seconds. + ("open", "d"), # Open float64 8 8 + ("high", "d"), # High float64 16 8 + ("low", "d"), # Low float64 24 8 + ("close", "d"), # Close float64 32 8 + ( + "volume", + "II", + ), # Volume uint64 40 8 Volume (documentation says it's a double, though it's stored as a long int). + ( + "tickTimestamp", + "i", + pretty_print_time, + ), # TickTimestamp uint32 48 4 Tick data timestamp in seconds (the current time within a bar). + ( + "launchExpert", + "i", + pretty_print_time, + ), # LaunchExpert uint32 52 4 Flag to launch an expert (0 - bar will be modified, but the expert will not be launched). + ] _truncate = True _size = get_fields_size(_fields) - assert(_size == 56) + assert _size == 56 + class HccHeader(BStruct): - _endianness = '<' + _endianness = "<" _fields = [ - ('magic', 'I'), - ('copyright', '128s', pretty_print_wstring), - ('name', '32s', pretty_print_wstring), - ('title', '64s', pretty_print_wstring) - ] + ("magic", "I"), + ("copyright", "128s", pretty_print_wstring), + ("name", "32s", pretty_print_wstring), + ("title", "64s", pretty_print_wstring), + ] _truncate = False _size = get_fields_size(_fields) - assert(_size == 228) + assert _size == 228 + class HccTable(BStruct): - _endianness = '<' + _endianness = "<" _fields = [ - ('unknown_0', 'I'), - ('unknown_1', 'I', pretty_print_time), - ('unknown_2', 'H'), - ('size', 'I'), - ('off', 'I', pretty_print_hex), - ] + ("unknown_0", "I"), + ("unknown_1", "I", pretty_print_time), + ("unknown_2", "H"), + ("size", "I"), + ("off", "I", pretty_print_hex), + ] _truncate = True _size = get_fields_size(_fields) - assert(_size == 18) + assert _size == 18 + class HccRecordHeader(BStruct): - _endianness = '<' + _endianness = "<" _fields = [ - ('magic', 'H'), - ('label', '64s', pretty_print_wstring), - ('unknown_0', '18s', pretty_print_ignore), - ('rows', 'I'), - ('unknown_1', '101s', pretty_print_ignore), - ] + ("magic", "H"), + ("label", "64s", pretty_print_wstring), + ("unknown_0", "18s", pretty_print_ignore), + ("rows", "I"), + ("unknown_1", "101s", pretty_print_ignore), + ] _truncate = True _size = get_fields_size(_fields) - assert(_size == 189) + assert _size == 189 + class HccRecord(BStruct): - _endianness = '<' + _endianness = "<" _fields = [ - ('separator', 'I', pretty_print_ignore), - ('time', 'I', pretty_print_time), - ('open', 'd'), - ('high', 'd'), - ('low', 'd'), - ('close', 'd'), - ] + ("separator", "I", pretty_print_ignore), + ("time", "I", pretty_print_time), + ("open", "d"), + ("high", "d"), + ("low", "d"), + ("close", "d"), + ] _size = get_fields_size(_fields) _truncate = True - assert(_size == 40) + assert _size == 40 + # Header structure for HST version 401. class HstHeader(BStruct): - _endianness = '<' + _endianness = "<" _fields = [ # Build header - ('headerVersion', 'I'), # Version uint32 // 0 4 HST version (default 401). - ('copyright', '64s', pretty_print_string), # Copyright [64]byte // 4 64 Copyright info. - ('symbol', '12s', pretty_print_string), # Symbol [12]byte // 68 12 Forex symbol. - ('timeframe', 'i'), # Period uint32 // 80 4 Symbol timeframe. - ('digits', 'I'), # Digits uint32 // 84 4 The amount of digits after decimal point in the symbol. - ('timeSign', 'I', pretty_print_time), # TimeSign uint32 // 88 4 Time of sign (database creation). - ('lastSync', 'I', pretty_print_time), # LastSync uint32 // 92 4 Time of last synchronization. - ('unused', '13s', pretty_print_bstring), # _ [13]uint32 // 96 52 Unused. - ] + ( + "headerVersion", + "I", + ), # Version uint32 // 0 4 HST version (default 401). + ( + "copyright", + "64s", + pretty_print_string, + ), # Copyright [64]byte // 4 64 Copyright info. + ( + "symbol", + "12s", + pretty_print_string, + ), # Symbol [12]byte // 68 12 Forex symbol. + ("timeframe", "i"), # Period uint32 // 80 4 Symbol timeframe. + ( + "digits", + "I", + ), # Digits uint32 // 84 4 The amount of digits after decimal point in the symbol. + ( + "timeSign", + "I", + pretty_print_time, + ), # TimeSign uint32 // 88 4 Time of sign (database creation). + ( + "lastSync", + "I", + pretty_print_time, + ), # LastSync uint32 // 92 4 Time of last synchronization. + ( + "unused", + "13s", + pretty_print_bstring, + ), # _ [13]uint32 // 96 52 Unused. + ] _size = get_fields_size(_fields) _truncate = False - assert(_size == 109) + assert _size == 109 + # HST bar data. # @see: https://www.metatrader4.com/en/trading-platform/help/autotrading/tester/tester_fxt class HstBar(BStruct): - _endianness = '<' + _endianness = "<" _fields = [ - ('barTimestamp', 'II', pretty_print_time), # CTM uint64 0 8 Current time in seconds aligned with timeframe (MQL4 datetime). - ('open', 'd'), # Open float64 8 8 - ('high', 'd'), # High float64 16 8 - ('low', 'd'), # Low float64 24 8 - ('close', 'd'), # Close float64 32 8 - ('volume', 'II'), # Volume uint64 40 8 Volume (documentation says it's a double, though it's stored as a long int). - ('spread', 'I'), # Spread uint32 48 8 Spread in points, 0 for current online spread (variable). - ('realVolume', 'II'), # Real volume uint64 52 8 - ] + ( + "barTimestamp", + "II", + pretty_print_time, + ), # CTM uint64 0 8 Current time in seconds aligned with timeframe (MQL4 datetime). + ("open", "d"), # Open float64 8 8 + ("high", "d"), # High float64 16 8 + ("low", "d"), # Low float64 24 8 + ("close", "d"), # Close float64 32 8 + ( + "volume", + "II", + ), # Volume uint64 40 8 Volume (documentation says it's a double, though it's stored as a long int). + ( + "spread", + "I", + ), # Spread uint32 48 8 Spread in points, 0 for current online spread (variable). + ("realVolume", "II"), # Real volume uint64 52 8 + ] _size = get_fields_size(_fields) _truncate = True - assert(_size == 60) + assert _size == 60 + class SrvHeader(BStruct): - _endianness = '<' + _endianness = "<" _fields = [ - ('serverName', '64s', pretty_print_string), - ('companyName', '128s', pretty_print_string), - ('unknown_0', '24s', pretty_print_bstring), - ('serverAddress', '64s', pretty_print_string), - ('unknown_1', '72s', pretty_print_bstring), - ] + ("serverName", "64s", pretty_print_string), + ("companyName", "128s", pretty_print_string), + ("unknown_0", "24s", pretty_print_bstring), + ("serverAddress", "64s", pretty_print_string), + ("unknown_1", "72s", pretty_print_bstring), + ] _size = get_fields_size(_fields) _truncate = False - assert(_size == 352) + assert _size == 352 + class SrvRecord(BStruct): - _endianness = '<' + _endianness = "<" _fields = [ - ('unknown_0', '40s', pretty_print_bstring), - ('unknown_1', '40s', pretty_print_bstring), - ('unknown_2', '40s', pretty_print_bstring), - ('unknown_3', '40s', pretty_print_bstring), - ] + ("unknown_0", "40s", pretty_print_bstring), + ("unknown_1", "40s", pretty_print_bstring), + ("unknown_2", "40s", pretty_print_bstring), + ("unknown_3", "40s", pretty_print_bstring), + ] _size = get_fields_size(_fields) _truncate = True - assert(_size == 160) + assert _size == 160 + # Experts.ini file (binary). MT4-only. class ExpertsIni(BStruct): - _endianness = '<' + _endianness = "<" _fields = [ - ('headerVersion', 'I'), # Version uint32 0 4 Header version (default: 400). - ('unknown_0', '44s'), # Unknown data [44]byte 4 47 Unknown data. - ('webRequestUrlEnabled', 'B'), # Enable URL white-listing [1]unsigned char 48 48 Boolean: 0/1. - ('unknown_1', '243s'), # Unknown data [245]byte 49 293 Unknown data. - ('webRequestUrl', '4096s', pretty_print_wstring), # Web Request URL [2080]byte 294 4387 Account server name (szchar). - ] + ( + "headerVersion", + "I", + ), # Version uint32 0 4 Header version (default: 400). + ( + "unknown_0", + "44s", + ), # Unknown data [44]byte 4 47 Unknown data. + ( + "webRequestUrlEnabled", + "B", + ), # Enable URL white-listing [1]unsigned char 48 48 Boolean: 0/1. + ( + "unknown_1", + "243s", + ), # Unknown data [245]byte 49 293 Unknown data. + ( + "webRequestUrl", + "4096s", + pretty_print_wstring, + ), # Web Request URL [2080]byte 294 4387 Account server name (szchar). + ] _truncate = True _size = get_fields_size(_fields) - assert(_size == 4388) + assert _size == 4388 diff --git a/src/fx-data-convert-from-csv.py b/src/fx-data-convert-from-csv.py index dd7cf85..cfedbc3 100755 --- a/src/fx-data-convert-from-csv.py +++ b/src/fx-data-convert-from-csv.py @@ -13,17 +13,20 @@ import sys import time + class Spinner: + """Displays an ASCII spinner""" + def __init__(self, step): self._n = self._x = 0 - self._chars = '\\|/-' + self._chars = "\\|/-" self._step = step def spin(self): self._n += 1 if self._n % self._step == 0: - sys.stdout.write('\b' + self._chars[self._x % 4]) + sys.stdout.write("\b" + self._chars[self._x % 4]) sys.stdout.flush() self._x += 1 @@ -33,57 +36,67 @@ def spin(self): self._n = 0 + spinner = Spinner(100000) + class Input: def __init__(self, path): if args.verbose: - print('[INFO] Trying to read data from %s...' % path) + print("[INFO] Trying to read data from %s..." % path) try: - self.path = open(path, 'r') + self.path = open(path, "r") except OSError as e: - print("[ERROR] '%s' raised when tried to read the file '%s'" % (e.strerror, e.filename)) + print( + "[ERROR] '%s' raised when tried to read the file '%s'" + % (e.strerror, e.filename) + ) sys.exit(1) + self.uniBars = [] def __del__(self): self.path.close() - def _addBar(self, barTimestamp, tickTimestamp, open, high, low, close, volume): - self.uniBars += [{ - 'barTimestamp': barTimestamp, - 'tickTimestamp': tickTimestamp, - 'open': open, - 'high': high, - 'low': low, - 'close': close, - 'volume': volume - }] + def _addBar( + self, barTimestamp, tickTimestamp, uniBar_open, high, low, close, volume + ): + self.uniBars += [ + { + "barTimestamp": barTimestamp, + "tickTimestamp": tickTimestamp, + "open": uniBar_open, + "high": high, + "low": low, + "close": close, + "volume": volume, + } + ] + def string_to_timestamp(s): try_microseconds = s[20:] - if try_microseconds == '': + if try_microseconds == "": microseconds = int(s[20:]) else: microseconds = 0 return datetime.datetime( - int(s[0:4]), # Year - int(s[5:7]), # Month - int(s[8:10]), # Day - int(s[11:13]), # Hour - int(s[14:16]), # Minute - int(s[17:19]), # Second - microseconds, # Microseconds - datetime.timezone.utc) + int(s[0:4]), # Year + int(s[5:7]), # Month + int(s[8:10]), # Day + int(s[11:13]), # Hour + int(s[14:16]), # Minute + int(s[17:19]), # Second + microseconds, # Microseconds + datetime.timezone.utc, + ) + class CSV(Input): def __init__(self, path): super().__init__(path) - if os.name == 'nt': - self._map_obj = mmap.mmap(self.path.fileno(), 0, access=mmap.ACCESS_READ) - else: - self._map_obj = mmap.mmap(self.path.fileno(), 0, prot=mmap.PROT_READ) + self._map_obj = mmap.mmap(self.path.fileno(), 0, access=mmap.ACCESS_READ) def __iter__(self): return self @@ -91,43 +104,47 @@ def __iter__(self): def __next__(self): line = self._map_obj.readline() if line: - isLastRow = self._map_obj.tell () == self._map_obj.size () + isLastRow = self._map_obj.tell() == self._map_obj.size() return (self._parseLine(line), isLastRow) - else: - raise StopIteration + raise StopIteration def _parseLine(self, line): - tick = line.split(b',') + tick = line.split(b",") return { # Storing timestamp as float to preserve its precision. # 'timestamp': time.mktime(datetime.datetime.strptime(tick[0], '%Y.%m.%d %H:%M:%S.%f').replace(tzinfo=datetime.timezone.utc).timetuple()), - 'timestamp': string_to_timestamp(tick[0]).timestamp(), - 'bidPrice': float(tick[1]), - 'askPrice': float(tick[2]), - 'bidVolume': float(tick[3]), - 'askVolume': float(tick[4]) # float() handles ending '\n' character + "timestamp": string_to_timestamp(tick[0]).timestamp(), + "bidPrice": float(tick[1]), + "askPrice": float(tick[2]), + "bidVolume": float(tick[3]), + "askVolume": float(tick[4]), # float() handles ending '\n' character } + class Output: def __init__(self, timeframe, path_suffix, symbol, output_dir): self.deltaTimestamp = timeframe * 60 self.endTimestamp = None self.barCount = 0 - self.filename = '%s%d%s' % (symbol, timeframe, path_suffix) + self.filename = "%s%d%s" % (symbol, timeframe, path_suffix) self.fullname = os.path.join(output_dir, self.filename) try: - os.remove(self.fullname) # Remove existing output file before creating an appended new one + os.remove( + self.fullname + ) # Remove existing output file before creating an appended new one except (OSError, IOError) as e: pass try: - self.path = open(self.fullname, 'wb') + self.path = open(self.fullname, "wb") except OSError as e: - print("[ERROR] '%s' raised when tried to open for appending the file '%s'" % (e.strerror, e.filename)) + print( + "[ERROR] '%s' raised when tried to open for appending the file '%s'" + % (e.strerror, e.filename) + ) sys.exit(1) - def __del__(self): self.path.close() @@ -135,62 +152,67 @@ def finalize(self): pass def _aggregate(self, tick): - if not self.endTimestamp or tick['timestamp'] >= self.endTimestamp: + if not self.endTimestamp or tick["timestamp"] >= self.endTimestamp: uniBar = None - if self.endTimestamp: uniBar = { - 'barTimestamp': self.startTimestamp, - 'tickTimestamp': tick['timestamp'], - 'open': self.open, - 'high': self.high, - 'low': self.low, - 'close': self.close, - 'volume': self.volume - } - - self.startTimestamp = (int(tick['timestamp']) // self.deltaTimestamp) * self.deltaTimestamp + if self.endTimestamp: + uniBar = { + "barTimestamp": self.startTimestamp, + "tickTimestamp": tick["timestamp"], + "open": self.open, + "high": self.high, + "low": self.low, + "close": self.close, + "volume": self.volume, + } + + self.startTimestamp = ( + int(tick["timestamp"]) // self.deltaTimestamp + ) * self.deltaTimestamp self.endTimestamp = self.startTimestamp + self.deltaTimestamp - self.open = self.high = self.low = self.close = tick['bidPrice'] - self.volume = tick['bidVolume'] + tick['askVolume'] + self.open = self.high = self.low = self.close = tick["bidPrice"] + self.volume = tick["bidVolume"] + tick["askVolume"] - if uniBar: return (uniBar, True) + if uniBar: + return (uniBar, True) else: - self.high = max(tick['bidPrice'], self.high) - self.low = min(tick['bidPrice'], self.low) - self.close = tick['bidPrice'] - self.volume += tick['bidVolume'] + tick['askVolume'] + self.high = max(tick["bidPrice"], self.high) + self.low = min(tick["bidPrice"], self.low) + self.close = tick["bidPrice"] + self.volume += tick["bidVolume"] + tick["askVolume"] uniBar = { - 'barTimestamp': self.startTimestamp, - 'tickTimestamp': tick['timestamp'], - 'open': self.open, - 'high': self.high, - 'low': self.low, - 'close': self.close, - 'volume': self.volume + "barTimestamp": self.startTimestamp, + "tickTimestamp": tick["timestamp"], + "open": self.open, + "high": self.high, + "low": self.low, + "close": self.close, + "volume": self.volume, } return (uniBar, False) - def _aggregateWithTicks(self, tick): - if not self.endTimestamp or tick['timestamp'] >= self.endTimestamp: - self.startTimestamp = (int(tick['timestamp']) // self.deltaTimestamp) * self.deltaTimestamp + if not self.endTimestamp or tick["timestamp"] >= self.endTimestamp: + self.startTimestamp = ( + int(tick["timestamp"]) // self.deltaTimestamp + ) * self.deltaTimestamp self.endTimestamp = self.startTimestamp + self.deltaTimestamp - self.open = self.high = self.low = tick['bidPrice'] - self.volume = tick['bidVolume'] + tick['askVolume'] + self.open = self.high = self.low = tick["bidPrice"] + self.volume = tick["bidVolume"] + tick["askVolume"] self.barCount += 1 else: - self.high = max(tick['bidPrice'], self.high) - self.low = min(tick['bidPrice'], self.low) - self.volume += tick['bidVolume'] + tick['askVolume'] + self.high = max(tick["bidPrice"], self.high) + self.low = min(tick["bidPrice"], self.low) + self.volume += tick["bidVolume"] + tick["askVolume"] return { - 'barTimestamp': self.startTimestamp, - 'tickTimestamp': tick['timestamp'], - 'open': self.open, - 'high': self.high, - 'low': self.low, - 'close': tick['bidPrice'], - 'volume': self.volume + "barTimestamp": self.startTimestamp, + "tickTimestamp": tick["timestamp"], + "open": self.open, + "high": self.high, + "low": self.low, + "close": tick["bidPrice"], + "volume": self.volume, } @@ -201,15 +223,20 @@ def __init__(self, path, path_suffix, output_dir, timeframe, symbol): # Build header (148 Bytes in total) header = bytearray() - header += pack(' highPrice: - highPrice = tick['bidPrice'] + elif tick["bidPrice"] > highPrice: + highPrice = tick["bidPrice"] self.write_unibar(tick) - elif tick['timestamp'] >= endTimestampTimeline: - startTimestamp = tick['barTimestamp'] + elif tick["timestamp"] >= endTimestampTimeline: + startTimestamp = tick["barTimestamp"] self.write_unibar(tick) # Open price model elif model == 2: @@ -425,18 +529,26 @@ def pack_ticks(self, ticks): def finalize(self): # Fixup the header. self.path.seek(216) - fix = bytearray() - fix += pack(' 12: - print('[WARNING] Symbol is more than 12 characters, cutting its end off!') + print("[WARNING] Symbol is more than 12 characters, cutting its end off!") symbol = args.pair[0:12] else: symbol = args.pair if args.verbose: - print('[INFO] Symbol pair name: %s' % symbol) + print("[INFO] Symbol pair name: %s" % symbol) # Converting timeframe argument to minutes. timeframe_list = [] timeframe_conv = { - 'M': 1, - 'H': 60, - 'D': 24 * 60, - 'W': 7 * 24 * 60, - 'MN': 30 * 24 * 60 + "M": 1, + "H": 60, + "D": 24 * 60, + "W": 7 * 24 * 60, + "MN": 30 * 24 * 60, } - for arg in args.timeframe.lower().split(','): - match_obj = re.match(r'(M|H|D|W|MN)(\d+)', arg, re.I) + for arg in args.timeframe.strip().upper().split(","): + match_obj = re.match(r"(M|H|D|W|MN)(\d+)", arg, re.I) if match_obj: model = match_obj.group(1).upper() value = int(match_obj.group(2)) - timeframe_list.append( - timeframe_conv[model] * value - ) + timeframe_list.append(timeframe_conv[model] * value) else: - print('[ERROR] Bad timeframe setting \'{}\'!'.format(arg)) + print("[ERROR] Bad timeframe setting '{}'!".format(arg)) sys.exit(1) if args.verbose: - print('[INFO] Timeframe: %s - %s minute(s)' % (args.timeframe.upper(), timeframe_list)) + print( + "[INFO] Timeframe: %s - %s minute(s)" + % (args.timeframe.upper(), timeframe_list) + ) # Checking spread argument spread = int(args.spread) if args.verbose: - print('[INFO] Spread: %d' % spread) + print("[INFO] Spread: %d" % spread) # Create output directory os.makedirs(args.outputDir, 0o755, True) if args.verbose: - print('[INFO] Output directory: %s' % args.outputDir) + print("[INFO] Output directory: %s" % args.outputDir) # Checking server argument if len(args.server) > 128: - print('[WARNING] Server name is longer than 128 characters, cutting its end off!') + print( + "[WARNING] Server name is longer than 128 characters, cutting its end off!" + ) server = args.server[0:128] else: server = args.server if args.verbose: - print('[INFO] Server name: %s' % server) + print("[INFO] Server name: %s" % server) outputFormat = args.outputFormat.strip().lower() if args.verbose: - print('[INFO] Output format: %s' % outputFormat) + print("[INFO] Output format: %s" % outputFormat) multiple_timeframes = len(timeframe_list) > 1 diff --git a/src/fx-data-convert-to-csv.py b/src/fx-data-convert-to-csv.py index 44e0f92..5e40681 100755 --- a/src/fx-data-convert-to-csv.py +++ b/src/fx-data-convert-to-csv.py @@ -9,34 +9,40 @@ import struct import sys + class Input: def __init__(self, fileName): if args.verbose: - print('[INFO] Trying to read data from %s...' % fileName) + print("[INFO] Trying to read data from %s..." % fileName) try: - with open(fileName, 'rb') as inputFile: + with open(fileName, "rb") as inputFile: self.content = inputFile.read() except OSError as e: - print("[ERROR] '%s' raised when tried to read the file '%s'" % (e.strerror, e.fileName)) + print( + "[ERROR] '%s' raised when tried to read the file '%s'" + % (e.strerror, e.fileName) + ) sys.exit(1) self._checkFormat() if self.rowLength != 0: - self.numberOfRows = (len(self.content) - self.headerLength)//self.rowLength + self.numberOfRows = ( + len(self.content) - self.headerLength + ) // self.rowLength self._parse() - def _checkFormat(self): - if (len(self.content) - self.headerLength)%self.rowLength != 0: - print('[ERROR] File length isn\'t valid for this kind of format!') + if (len(self.content) - self.headerLength) % self.rowLength != 0: + print("[ERROR] File length isn't valid for this kind of format!") - if self.version != struct.unpack('> 28) & 15) + - ((tick.separator >> 24) & 15) + - ((tick.separator >> 20) & 15) + ((tick.separator >> 28) & 15) + + ((tick.separator >> 24) & 15) + + ((tick.separator >> 20) & 15) ) base += HccTable._size def __str__(self): - table = '' - separator = ',' + table = "" + separator = "," for row in self.rows: - table += '{:<19}'.format('{:%Y.%m.%d %H:%M:%S}'.format(row['timestamp'])) + table += "{:<19}".format("{:%Y.%m.%d %H:%M:%S}".format(row["timestamp"])) table += separator - table += '{:>9.5f}'.format(row['open']) + table += "{:>9.5f}".format(row["open"]) table += separator - table += '{:>9.5f}'.format(row['high']) + table += "{:>9.5f}".format(row["high"]) table += separator - table += '{:>9.5f}'.format(row['low']) + table += "{:>9.5f}".format(row["low"]) table += separator - table += '{:>9.5f}'.format(row['close']) - table += '\n' + table += "{:>9.5f}".format(row["close"]) + table += "\n" return table[:-1] - def toCsv(self, fileName): - with open(fileName, 'w', newline='') as csvFile: - writer = csv.writer(csvFile, quoting = csv.QUOTE_NONE) + with open(fileName, "w", newline="") as csvFile: + writer = csv.writer(csvFile, quoting=csv.QUOTE_NONE) for row in self.rows: - writer.writerow(['{:%Y.%m.%d %H:%M:%S}'.format(row['timestamp']), - '{:.5f}'.format(row['open']), - '{:.5f}'.format(row['high']), - '{:.5f}'.format(row['low']), - '{:.5f}'.format(row['close']), - ]) + writer.writerow( + [ + "{:%Y.%m.%d %H:%M:%S}".format(row["timestamp"]), + "{:.5f}".format(row["open"]), + "{:.5f}".format(row["high"]), + "{:.5f}".format(row["low"]), + "{:.5f}".format(row["close"]), + ] + ) + class HST509(Input): version = 400 @@ -126,47 +137,63 @@ class HST509(Input): def _parse(self): self.rows = [] for i in range(0, self.numberOfRows): - base = self.headerLength + i*self.rowLength - self.rows += [{'timestamp': datetime.datetime.fromtimestamp( - struct.unpack('