From 0fb23cf4c7916b74175aa499532e681310c0cc07 Mon Sep 17 00:00:00 2001 From: sina <20732540+SinaKhalili@users.noreply.github.com> Date: Fri, 11 Oct 2024 14:10:46 -0700 Subject: [PATCH] Add examples for getting data in range for trading --- Gemfile | 4 +- Gemfile.lock | 10 +- source/includes/_historicaldata.md | 171 ++++++++++++++++++++++++++++- 3 files changed, 175 insertions(+), 10 deletions(-) diff --git a/Gemfile b/Gemfile index f2e8441..35d76cf 100644 --- a/Gemfile +++ b/Gemfile @@ -1,4 +1,4 @@ -ruby '>= 2.6' +ruby '>= 3' source 'https://rubygems.org' # Middleman @@ -8,6 +8,6 @@ gem 'middleman-autoprefixer', '~> 3.0' gem 'middleman-sprockets', '~> 4.1' gem 'rouge', '~> 3.21' gem 'redcarpet', '~> 3.6.0' -gem 'nokogiri', '~> 1.13.3' +gem 'nokogiri', '~> 1.16.1' gem 'sass' gem 'webrick' diff --git a/Gemfile.lock b/Gemfile.lock index 79bfbcb..c9cf4e4 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -80,10 +80,8 @@ GEM middleman-syntax (3.3.0) middleman-core (>= 3.2) rouge (~> 3.2) - mini_portile2 (2.8.0) minitest (5.17.0) - nokogiri (1.13.9) - mini_portile2 (~> 2.8.0) + nokogiri (1.16.7-arm64-darwin) racc (~> 1.4) padrino-helpers (0.15.2) i18n (>= 0.6.7, < 2) @@ -93,7 +91,7 @@ GEM parallel (1.22.1) parslet (2.0.0) public_suffix (5.0.1) - racc (1.6.0) + racc (1.8.1) rack (2.2.6.2) rb-fsevent (0.11.2) rb-inotify (0.10.1) @@ -132,14 +130,14 @@ DEPENDENCIES middleman-autoprefixer (~> 3.0) middleman-sprockets (~> 4.1) middleman-syntax (~> 3.2) - nokogiri (~> 1.13.3) + nokogiri (~> 1.16.1) redcarpet (~> 3.6.0) rouge (~> 3.21) sass webrick RUBY VERSION - ruby 2.7.2p137 + ruby 3.3.5p100 BUNDLED WITH 2.2.22 diff --git a/source/includes/_historicaldata.md b/source/includes/_historicaldata.md index 6857e64..712b465 100644 --- a/source/includes/_historicaldata.md +++ b/source/includes/_historicaldata.md @@ -40,13 +40,180 @@ mainnet-beta: `https://drift-historical-data-v2.s3.eu-west-1.amazonaws.com/prog | candleResolution | | 1M | -## Examples +## Example: Trades for market +```python +import requests +import csv +from io import StringIO + +URL_PREFIX = 'https://drift-historical-data-v2.s3.eu-west-1.amazonaws.com/program/dRiftyHA39MWEi3m9aunc5MzRF1JYuBsbn6VPcn33UH' + +market_symbol = 'SOL-PERP' +year = '2024' +month = '01' +day = '01' +response = requests.get(f'{URL_PREFIX}/market/{market_symbol}/tradeRecords/{year}/{year}{month}{day}') +response.raise_for_status() + +csv_data = StringIO(response.text) +reader = csv.reader(csv_data) +for row in reader: + print(row) +``` + +```typescript +import { parse } from 'csv-parse/sync'; + +const URL_PREFIX = 'https://drift-historical-data-v2.s3.eu-west-1.amazonaws.com/program/dRiftyHA39MWEi3m9aunc5MzRF1JYuBsbn6VPcn33UH'; + +async function fetchAndParseCSV(marketSymbol: string, year: string, month: string, day: string): Promise { + const url = `${URL_PREFIX}/market/${marketSymbol}/tradeRecords/${year}/${year}${month}${day}`; + + try { + const response = await fetch(url); + if (!response.ok) { + throw new Error(`HTTP error! status: ${response.status}`); + } + + const csvText = await response.text(); + const records = parse(csvText, { + skip_empty_lines: true + }); + + records.forEach((row: string[]) => { + console.log(row); + }); + } catch (error) { + console.error('Error fetching or parsing CSV:', error); + } +} -Get historical trades on `SOL-PERP` for August 5, 2023: +async function main() { + const marketSymbol = 'SOL-PERP'; + const year = '2024'; + const month = '01'; + const day = '01'; + + await fetchAndParseCSV(marketSymbol, year, month, day); +} + +main().catch(error => console.error('An error occurred:', error)); +``` + +Get historical trades on `SOL-PERP` for a given date (ex. 2024-01-01): ``` https://drift-historical-data-v2.s3.eu-west-1.amazonaws.com/program/dRiftyHA39MWEi3m9aunc5MzRF1JYuBsbn6VPcn33UH/market/SOL-PERP/tradeRecords/2024/20240101 ``` + +## Example: Trades for date range +```python +import requests +import csv +from io import StringIO +from datetime import date, timedelta + +URL_PREFIX = 'https://drift-historical-data-v2.s3.eu-west-1.amazonaws.com/program/dRiftyHA39MWEi3m9aunc5MzRF1JYuBsbn6VPcn33UH' + +def get_trades_for_range(account_key, start_date, end_date): + all_trades = [] + current_date = start_date + while current_date <= end_date: + year = current_date.year + month = current_date.month + day = current_date.day + url = f"{URL_PREFIX}/user/{account_key}/tradeRecords/{year}/{year}{month:02}{day:02}" + response = requests.get(url) + response.raise_for_status() + + csv_data = StringIO(response.text) + reader = csv.reader(csv_data) + for row in reader: + all_trades.append(row) + + current_date += timedelta(days=1) + + return all_trades + + +# Example usage +account_key = "" +start_date = date(2024, 1, 24) +end_date = date(2024, 1, 26) + +trades = get_trades_for_range(account_key, start_date, end_date) +``` + +```typescript +import { parse } from 'csv-parse/sync'; + +const URL_PREFIX = 'https://drift-historical-data-v2.s3.eu-west-1.amazonaws.com/program/dRiftyHA39MWEi3m9aunc5MzRF1JYuBsbn6VPcn33UH'; + +interface Trade { + // Define the structure of a trade record here + // For example: + // date: string; + // amount: number; + // price: number; + [key: string]: any; +} + +async function getTradesForRange(accountKey: string, startDate: Date, endDate: Date): Promise { + const allTrades: Trade[] = []; + let currentDate = new Date(startDate); + + while (currentDate <= endDate) { + const year = currentDate.getFullYear(); + const month = (currentDate.getMonth() + 1).toString().padStart(2, '0'); + const day = currentDate.getDate().toString().padStart(2, '0'); + + const url = `${URL_PREFIX}/user/${accountKey}/tradeRecords/${year}/${year}${month}${day}`; + + try { + const response = await fetch(url); + if (!response.ok) { + throw new Error(`HTTP error! status: ${response.status}`); + } + + const csvText = await response.text(); + const records = parse(csvText, { + skip_empty_lines: true, + columns: true + }); + + allTrades.push(...records); + } catch (error) { + console.error(`Error fetching or parsing CSV for ${year}-${month}-${day}:`, error); + } + + currentDate.setDate(currentDate.getDate() + 1); + } + + return allTrades; +} + +async function main() { + const accountKey = ""; + const startDate = new Date(2024, 0, 24); // Note: month is 0-indexed in JavaScript + const endDate = new Date(2024, 0, 26); + + try { + const trades = await getTradesForRange(accountKey, startDate, endDate); + console.log(`Total trades fetched: ${trades.length}`); + console.log('First few trades:', trades.slice(0, 5)); + } catch (error) { + console.error('An error occurred:', error); + } +} + +main(); +``` + +> Note: To speed this up, you could download the data in parallel (Promise.all or asyncio). + +We can write a script to download all the data, one by one, for a given account in a given date range. + + ## Records Columns Below are definitions of the columns in each record type.