Release 1.180.0

See release notes.
nautechsystems · Nov 3, 2023 · 622014e · 622014e
2 parents ef4d72e + 7db5619
commit 622014e
Show file tree

Hide file tree

Showing 337 changed files with 6,421 additions and 3,459 deletions.
diff --git a/.deepsource.toml b/.deepsource.toml
@@ -0,0 +1,16 @@
+version = 1
+
+[[analyzers]]
+name = "rust"
+
+  [analyzers.meta]
+  msrv = "stable"
+
+[[analyzers]]
+name = "shell"
+
+[[analyzers]]
+name = "python"
+
+  [analyzers.meta]
+  runtime_version = "3.x.x"
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -15,7 +15,7 @@ jobs:
       matrix:
         arch: [x64]
         os: [ubuntu-latest, macos-latest, windows-latest]
-        python-version: ["3.9", "3.10", "3.11"]
+        python-version: ["3.10", "3.11"]
     defaults:
       run:
         shell: bash
@@ -89,7 +89,9 @@ jobs:
           key: ${{ runner.os }}-${{ matrix.python-version }}-poetry-${{ hashFiles('**/poetry.lock') }}
 
       - name: Run pre-commit
-        run: pre-commit run --all-files
+        run: |
+          # pre-commit run --hook-stage manual gitlint-ci
+          pre-commit run --all-files
 
       - name: Install Redis (macOS)
         if: runner.os == 'macOS'

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -18,7 +18,7 @@ jobs:
       matrix:
         arch: [x64]
         os: [ubuntu-latest, macos-latest, windows-latest]
-        python-version: ["3.9", "3.10", "3.11"]
+        python-version: ["3.10", "3.11"]
     defaults:
       run:
         shell: bash
@@ -245,7 +245,7 @@ jobs:
       matrix:
         arch: [x64]
         os: [ubuntu-20.04, ubuntu-latest, windows-latest]
-        python-version: ["3.9", "3.10", "3.11"]
+        python-version: ["3.10", "3.11"]
     defaults:
       run:
         shell: bash
@@ -318,9 +318,9 @@ jobs:
       - name: Set release output
         id: vars
         run: |
-          echo "ASSET_PATH=$(find ./dist -mindepth 1 -print -quit)" >> $ GITHUB_ENV
+          echo "ASSET_PATH=$(find ./dist -mindepth 1 -print -quit)" >> $GITHUB_ENV
           cd dist
-          echo "ASSET_NAME=$(printf '%s\0' * | awk 'BEGIN{RS="\0"} {print;  exit}')" >> $ GITHUB_ENV
+          echo "ASSET_NAME=$(printf '%s\0' * | awk 'BEGIN{RS="\0"} {print;  exit}')" >> $GITHUB_ENV
 
       - name: Upload release asset
         id: upload-release-asset-unix

diff --git a/.gitlint b/.gitlint
@@ -0,0 +1,8 @@
+[general]
+ignore=body-is-missing,body-min-length
+
+[title-max-length]
+line-length=80
+
+[title-min-length]
+min-length=5
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -24,6 +24,12 @@ repos:
       - id: check-xml
       - id: check-yaml
 
+  - repo: https://github.com/jorisroovers/gitlint
+    rev: v0.19.1
+    hooks:
+      - id: gitlint
+      # - id: gitlint-ci
+
   - repo: https://github.com/codespell-project/codespell
     rev: v2.2.6
     hooks:
@@ -67,7 +73,7 @@ repos:
         types: [python]
 
   - repo: https://github.com/psf/black
-    rev: 23.10.0
+    rev: 23.10.1
     hooks:
       - id: black
         types_or: [python, pyi]
@@ -76,7 +82,7 @@ repos:
         exclude: "docs/_pygments/monokai.py"
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.1.1
+    rev: v0.1.3
     hooks:
       - id: ruff
         args: ["--fix"]

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -16,7 +16,7 @@ To contribute, follow these steps:
 6. The CI system will run the full test-suite over your code including all unit and integration tests, so include appropriate tests
   with the PR.
 
-7. [Codacy](https://www.codacy.com/) will perform an automated code review.
+7. [Deepsource](https://deepsource.io) will perform an automated code review.
   Fix any issues which cause a failed check, and add the commit to your PR.
 
 8. You will also be required to sign a standard Contributor License Agreement (CLA), which is administered automatically through [CLA Assistant](https://cla-assistant.io/).

diff --git a/README.md b/README.md
@@ -1,6 +1,5 @@
 # <img src="https://github.com/nautechsystems/nautilus_trader/blob/develop/docs/_images/nautilus-trader-logo.png" width="500">
 
-[![codacy-quality](https://api.codacy.com/project/badge/Grade/a1d3ccf7bccb4483b091975681a5cb23)](https://app.codacy.com/gh/nautechsystems/nautilus_trader?utm_source=github.com&utm_medium=referral&utm_content=nautechsystems/nautilus_trader&utm_campaign=Badge_Grade_Dashboard)
 [![codecov](https://codecov.io/gh/nautechsystems/nautilus_trader/branch/master/graph/badge.svg?token=DXO9QQI40H)](https://codecov.io/gh/nautechsystems/nautilus_trader)
 ![pythons](https://img.shields.io/pypi/pyversions/nautilus_trader)
 ![pypi-version](https://img.shields.io/pypi/v/nautilus_trader)
@@ -15,10 +14,10 @@
 
 | Platform           | Rust    | Python |
 | :----------------- | :------ | :----- |
-| `Linux (x86_64)`   | 1.73.0+ | 3.9+   |
-| `macOS (x86_64)`   | 1.73.0+ | 3.9+   |
-| `macOS (arm64)`    | 1.73.0+ | 3.9+   |
-| `Windows (x86_64)` | 1.73.0+ | 3.9+   |
+| `Linux (x86_64)`   | 1.73.0+ | 3.10+  |
+| `macOS (x86_64)`   | 1.73.0+ | 3.10+  |
+| `macOS (arm64)`    | 1.73.0+ | 3.10+  |
+| `Windows (x86_64)` | 1.73.0+ | 3.10+  |
 
 - **Website:** https://nautilustrader.io
 - **Docs:** https://docs.nautilustrader.io
@@ -140,14 +139,15 @@ NautilusTrader is designed in a modular way to work with 'adapters' which provid
 connectivity to data publishers and/or trading venues - converting their raw API
 into a unified interface. The following integrations are currently supported:
 
-| Name                                                      | ID        | Type                    | Status                                                  | Docs                                                              |
-| :-------------------------------------------------------- | :-------- | :---------------------- | :------------------------------------------------------ | :---------------------------------------------------------------- |
-| [Betfair](https://betfair.com)                            | `BETFAIR` | Sports Betting Exchange | ![status](https://img.shields.io/badge/beta-yellow)     | [Guide](https://docs.nautilustrader.io/integrations/betfair.html) |
-| [Binance](https://binance.com)                            | `BINANCE` | Crypto Exchange (CEX)   | ![status](https://img.shields.io/badge/stable-green)    | [Guide](https://docs.nautilustrader.io/integrations/binance.html) |
-| [Binance US](https://binance.us)                          | `BINANCE` | Crypto Exchange (CEX)   | ![status](https://img.shields.io/badge/stable-green)    | [Guide](https://docs.nautilustrader.io/integrations/binance.html) |
-| [Binance Futures](https://www.binance.com/en/futures)     | `BINANCE` | Crypto Exchange (CEX)   | ![status](https://img.shields.io/badge/stable-green)    | [Guide](https://docs.nautilustrader.io/integrations/binance.html) |
-| [Bybit](https://www.bybit.com)                            | `BYBIT`   | Crypto Exchange (CEX)   | ![status](https://img.shields.io/badge/building-orange) |                                                                   |
-| [Interactive Brokers](https://www.interactivebrokers.com) | `IB`      | Brokerage (multi-venue) | ![status](https://img.shields.io/badge/beta-yellow)     | [Guide](https://docs.nautilustrader.io/integrations/ib.html)      |
+| Name                                                      | ID          | Type                    | Status                                                  | Docs                                                              |
+| :-------------------------------------------------------- | :---------- | :---------------------- | :------------------------------------------------------ | :---------------------------------------------------------------- |
+| [Betfair](https://betfair.com)                            | `BETFAIR`   | Sports betting exchange | ![status](https://img.shields.io/badge/beta-yellow)     | [Guide](https://docs.nautilustrader.io/integrations/betfair.html) |
+| [Binance](https://binance.com)                            | `BINANCE`   | Crypto exchange (CEX)   | ![status](https://img.shields.io/badge/stable-green)    | [Guide](https://docs.nautilustrader.io/integrations/binance.html) |
+| [Binance US](https://binance.us)                          | `BINANCE`   | Crypto exchange (CEX)   | ![status](https://img.shields.io/badge/stable-green)    | [Guide](https://docs.nautilustrader.io/integrations/binance.html) |
+| [Binance Futures](https://www.binance.com/en/futures)     | `BINANCE`   | Crypto exchange (CEX)   | ![status](https://img.shields.io/badge/stable-green)    | [Guide](https://docs.nautilustrader.io/integrations/binance.html) |
+| [Bybit](https://www.bybit.com)                            | `BYBIT`     | Crypto exchange (CEX)   | ![status](https://img.shields.io/badge/building-orange) |                                                                   |
+| [Databento](https://databento.com)                        | `DATABENTO` | Data provider           | ![status](https://img.shields.io/badge/building-orange) |                                                                   |
+| [Interactive Brokers](https://www.interactivebrokers.com) | `IB`        | Brokerage (multi-venue) | ![status](https://img.shields.io/badge/beta-yellow)     | [Guide](https://docs.nautilustrader.io/integrations/ib.html)      |
 
 Refer to the [Integrations](https://docs.nautilustrader.io/integrations/index.html) documentation for further details.
 

diff --git a/RELEASES.md b/RELEASES.md
@@ -1,3 +1,44 @@
+# NautilusTrader 1.180.0 Beta
+
+Released on 3rd November 2023 (UTC).
+
+### Enhancements
+- Improved internal latency for live engines by using `loop.call_soon_threadsafe(...)`
+- Improved `RedisCacheDatabase` client connection error handling with retries
+- Added `WebSocketClient` connection headers, thanks @ruthvik125 and @twitu
+- Added `support_contingent_orders` option for venues (to simulate venues which do not support contingent orders)
+- Added `StrategyConfig.manage_contingent_orders` option (to automatically manage **open** contingenct orders)
+- Added `FuturesContract.activation_utc` property which returns a `pd.Timestamp` tz-aware (UTC)
+- Added `OptionsContract.activation_utc` property which returns a `pd.Timestamp` tz-aware (UTC)
+- Added `CryptoFuture.activation_utc` property which returns a `pd.Timestamp` tz-aware (UTC)
+- Added `FuturesContract.expiration_utc` property which returns a `pd.Timestamp` tz-aware (UTC)
+- Added `OptionsContract.expiration_utc` property which returns a `pd.Timestamp` tz-aware (UTC)
+- Added `CryptoFuture.expiration_utc` property which returns a `pd.Timestamp` tz-aware (UTC)
+
+### Breaking Changes
+- Renamed `FuturesContract.expiry_date` to `expiration_ns` (and associated params) as `uint64_t` UNIX nanoseconds
+- Renamed `OptionsContract.expiry_date` to `expiration_ns` (and associated params) as `uint64_t` UNIX nanoseconds
+- Renamed `CryptoFuture.expiry_date` to `expiration_ns` (and associated params) as `uint64_t` UNIX nanoseconds
+- Changed `FuturesContract` arrow schema
+- Changed `OptionsContract` arrow schema
+- Changed `CryptoFuture` arrow schema
+- Transformed orders will now retain the original `ts_init` timestamp
+- Removed unimplemented `batch_more` option for `Strategy.modify_order`
+- Removed `InstrumentProvider.venue` property (redundant as a provider may have many venues)
+- Dropped support for Python 3.9
+
+### Fixes
+- Fixed `ParquetDataCatalog` file writing template, thanks @limx0
+- Fixed `Binance` all orders requests which would omit order reports when using a `start` param
+- Fixed managed GTD orders past expiry cancellation on restart (orders were not being canceled)
+- Fixed managed GTD orders cancel timer on order cancel (timers were not being canceled)
+- Fixed `BacktestEngine` logging error with immediate stop (caused by certain timestamps being `None`)
+- Fixed `BacktestNode` exceptions during backtest runs preventing next sequential run, thanks for reporting @cavan-black
+- Fixed `BinanceSpotPersmission` value error by relaxing typing for `BinanceSpotSymbolInfo.permissions`
+- Interactive Brokers adapter various fixes, thanks @rsmb7z
+
+---
+
 # NautilusTrader 1.179.0 Beta
 
 Released on 22nd October 2023 (UTC).
@@ -153,7 +194,7 @@ Released on 31st July 2023 (UTC).
 - Fixed dictionary representation of orders for `venue_order_id` (for three order types)
 - Fixed `Currency` registration with core global map on creation
 - Fixed serialization of `OrderInitialized.exec_algorithm_params` to spec (bytes rather than string)
-- Fixed assignment of position IDs for contingency orders (when parent filled)
+- Fixed assignment of position IDs for contingent orders (when parent filled)
 - Fixed `PENDING_CANCEL` -> `EXPIRED` as valid state transition (real world possibility)
 - Fixed fill handling of `reduce_only` orders when partially filled
 - Fixed Binance reconciliation which was requesting reports for the same symbol multiple times
@@ -231,8 +272,8 @@ Released on 19th May 2023 (UTC).
 - Fixed handling of emulated order contingencies (not based on status of spawned algorithm orders)
 - Fixed sending execution algorithm commands from strategy
 - Fixed `OrderEmulator` releasing of already closed orders
-- Fixed `MatchingEngine` processing of reduce only for child contingency orders
-- Fixed `MatchingEngine` position ID assignment for child contingency orders
+- Fixed `MatchingEngine` processing of reduce only for child contingent orders
+- Fixed `MatchingEngine` position ID assignment for child contingent orders
 - Fixed `Actor` handling of historical data from requests (will now call `on_historical_data` regardless of state), thanks for reporting @miller-moore
 - Fixed pyarrow schema dictionary index keys being too narrow (int8 -> int16), thanks for reporting @rterbush
 
@@ -279,15 +320,15 @@ Released on 30th April 2023 (UTC).
 - Added `TWAPExecAlgorithm` and `TWAPExecAlgorithmConfig` to examples
 - Build out `ExecAlgorithm` base class for implementing 'first class' execution algorithms
 - Rewired execution for improved flow flexibility between emulated orders, execution algorithms and the `RiskEngine`
-- Improved handling for `OrderEmulator` updating of contingency orders from execution algorithms
+- Improved handling for `OrderEmulator` updating of contingent orders from execution algorithms
 - Defined public API for instruments, can now import directly from `nautilus_trader.model.instruments` (denest namespace)
 - Defined public API for orders, can now import directly from `nautilus_trader.model.orders` (denest namespace)
 - Defined public API for order book, can now import directly from `nautilus_trader.model.orderbook` (denest namespace)
 - Now stripping debug symbols after build (reduced binary sizes)
 - Refined build and added additional `debug` Makefile convenience targets
 
 ### Fixes
-- Fixed processing of contingency orders when in a pending update state
+- Fixed processing of contingent orders when in a pending update state
 - Fixed calculation of PnL for flipped positions (only book realized PnL against open position)
 - Fixed `WebSocketClient` session disconnect, thanks for reporting @miller-moore
 - Added missing `BinanceSymbolFilterType.NOTIONAL`
@@ -596,7 +637,7 @@ Released on 28th November 2022 (UTC).
 - Renamed `Instrument.get_cost_currency(...)` to `Instrument.get_settlement_currency(...)` (more accurate terminology)
 
 ### Enhancements
-- Added emulated contingency orders capability to `OrderEmulator`
+- Added emulated contingent orders capability to `OrderEmulator`
 - Moved `test_kit` module to main package to support downstream project/package testing
 
 ### Fixes
@@ -628,7 +669,7 @@ Released on 18th November 2022 (UTC).
 - Fixed bar aggregation start times for bar specs outside typical intervals (60-SECOND rather than 1-MINUTE etc) 
 - Fixed backtest engine main loop ordering of time events with identically timestamped data
 - Fixed `ModifyOrder` message `str` and `repr` when no quantity
-- Fixed OCO contingency orders which were actually implemented as OUO for backtests
+- Fixed OCO contingent orders which were actually implemented as OUO for backtests
 - Fixed various bugs for Interactive Brokers integration, thanks @limx0 and @rsmb7z
 - Fixed pyarrow version parsing, thanks @ghill2
 - Fixed returning venue from InstrumentId, thanks @rsmb7z
@@ -1444,7 +1485,7 @@ Released on 12th September 2021.
 - Added order custom user tags
 - Added `Actor.register_warning_event` (also applicable to `TradingStrategy`)
 - Added `Actor.deregister_warning_event` (also applicable to `TradingStrategy`)
-- Added `ContingencyType` enum (for contingency orders in an `OrderList`)
+- Added `ContingencyType` enum (for contingent orders in an `OrderList`)
 - All order types can now be `reduce_only` (#437)
 - Refined backtest configuration options
 - Improved efficiency of `UUID4` using the Rust `fastuuid` Python bindings

diff --git a/build.py b/build.py
@@ -1,7 +1,5 @@
 #!/usr/bin/env python3
 
-from __future__ import annotations
-
 import itertools
 import os
 import platform
@@ -50,9 +48,6 @@
     # Use clang as the default compiler
     os.environ["CC"] = "clang"
     os.environ["LDSHARED"] = "clang -shared"
-# elif platform.system() == "Windows":
-#     os.environ["CC"] = "cl"
-#     os.environ["CXX"] = "cl"
 
 TARGET_DIR = Path.cwd() / "nautilus_core" / "target" / BUILD_MODE
 
@@ -148,15 +143,8 @@ def _build_extensions() -> list[Extension]:
     extra_compile_args = []
     extra_link_args = RUST_LIBS
 
-    if platform.system() == "Darwin":
-        extra_compile_args.append("-Wno-unreachable-code-fallthrough")
-        extra_link_args.append("-flat_namespace")
-        extra_link_args.append("-undefined")
-        extra_link_args.append("suppress")
-
     if platform.system() != "Windows":
         # Suppress warnings produced by Cython boilerplate
-        extra_compile_args.append("-Wno-parentheses-equality")
         extra_compile_args.append("-Wno-unreachable-code")
         if BUILD_MODE == "release":
             extra_compile_args.append("-O2")

diff --git a/docs/concepts/adapters.md b/docs/concepts/adapters.md
@@ -62,6 +62,8 @@ as configured:
 - All instruments are automatically loaded on start:
 
 ```python
+from nautilus_trader.config import InstrumentProviderConfig
+
 InstrumentProviderConfig(load_all=True)
 ```
 
@@ -124,6 +126,10 @@ cpdef void request_instrument(self, InstrumentId instrument_id, ClientId client_
 The handler on the `ExecutionClient`:
 
 ```python
+from nautilus_trader.core.uuid import UUID4
+from nautilus_trader.model.data import DataType
+from nautilus_trader.model.identifiers import InstrumentId
+
 # nautilus_trader/adapters/binance/spot/data.py
 def request_instrument(self, instrument_id: InstrumentId, correlation_id: UUID4):
     instrument: Optional[Instrument] = self._instrument_provider.find(instrument_id)