part1 fixes

This commit is contained in:
2026-06-22 17:58:42 -07:00
parent 669271901d
commit dc22ebd3fa
4 changed files with 150 additions and 12 deletions
+2
View File
@@ -2,3 +2,5 @@
__pycache__/
*.pyc
*.pyo
cache/
*.parquet
-1
View File
@@ -7,7 +7,6 @@ requires-python = ">=3.14"
dependencies = [
"aiohttp>=3.14.1",
"altair>=6.2.1",
"binance>=0.3.110",
"cryptography>=49.0.0",
"jupyter>=1.1.1",
"matplotlib>=3.10.9",
@@ -0,0 +1,148 @@
from typing import List
import requests
import zipfile
from pathlib import Path
import polars as pl
from datetime import datetime, timedelta
from tqdm import tqdm
import research
MAKER_FEE = 0.000450
TAKER_FEE = 0.000450
def download_and_unzip(symbol: str, date: str | datetime,
download_dir: str = "data", cache_dir: str = "cache") -> pl.DataFrame:
"""
Download and unzip Binance futures trade data for a given symbol and date.
Caches results as parquet files to avoid repeated downloads.
"""
# Normalize date to string
date_str = date.strftime('%Y-%m-%d') if isinstance(date, datetime) else date
cache_dir = Path(cache_dir)
cache_dir.mkdir(exist_ok=True)
cache_path = cache_dir / f"{symbol}-trades-{date_str}.parquet"
if cache_path.exists():
return pl.read_parquet(cache_path)
url = f"https://data.binance.vision/data/futures/um/daily/trades/{symbol}/{symbol}-trades-{date_str}.zip"
download_dir = Path(download_dir)
download_dir.mkdir(exist_ok=True)
zip_path = download_dir / f"{symbol}-trades-{date_str}.zip"
# Download zip
response = requests.get(url, stream=True)
response.raise_for_status()
with open(zip_path, 'wb') as f:
for chunk in response.iter_content(chunk_size=8192):
f.write(chunk)
# Extract
with zipfile.ZipFile(zip_path, 'r') as zf:
zf.extractall(download_dir)
csv_path = download_dir / f"{symbol}-trades-{date_str}.csv"
# Load into Polars
df = pl.read_csv(
csv_path,
schema={
"id": pl.Int64,
"price": pl.Float64,
"qty": pl.Float64,
"quoteQty": pl.Float64,
"time": pl.Int64,
"isBuyerMaker": pl.Boolean,
}
).with_columns(
pl.from_epoch("time", time_unit="ms").alias("datetime")
)
# Cache and clean
df.write_parquet(cache_path)
zip_path.unlink(missing_ok=True)
csv_path.unlink(missing_ok=True)
return df
def download_date_range(symbol: str, start_date: str | datetime, end_date: str | datetime,
download_dir: str = "data", cache_dir: str = "cache") -> list[pl.DataFrame]:
"""
Download trade data for a range of dates with a progress bar.
"""
if isinstance(start_date, str):
start_date = datetime.strptime(start_date, '%Y-%m-%d')
if isinstance(end_date, str):
end_date = datetime.strptime(end_date, '%Y-%m-%d')
num_days = (end_date - start_date).days + 1
for i in tqdm(range(num_days), desc=f"Downloading {symbol}"):
current_date = start_date + timedelta(days=i)
try:
download_and_unzip(symbol, current_date, download_dir, cache_dir)
except Exception as e:
tqdm.write(f"[ERROR] {symbol} {current_date.date()}: {e}")
def download_trades(symbol: str, no_days: int,
download_dir: str = "data", cache_dir: str = "cache", return_trades=False) -> pl.DataFrame:
"""
Download trades for the last N days up to yesterday with a progress bar.
"""
yesterday = datetime.now() - timedelta(days=1)
start_date = yesterday - timedelta(days=no_days - 1)
dfs = []
for i in tqdm(range(no_days), desc=f"Downloading {symbol}"):
current_date = start_date + timedelta(days=i)
try:
if return_trades:
dfs.append(download_and_unzip(symbol, current_date, download_dir, cache_dir))
else:
download_and_unzip(symbol, current_date, download_dir, cache_dir)
except Exception as e:
tqdm.write(f"[ERROR] {symbol} {current_date.date()}: {e}")
return pl.concat(dfs) if return_trades else None
def download_ohlc_timeseries(symbol: str, no_days: int, time_interval: str, download_dir: str = "data", cache_dir: str = "cache") -> pl.DataFrame:
"""
Download trades for the last N days up to yesterday with a progress bar.
"""
yesterday = datetime.now() - timedelta(days=1)
start_date = yesterday - timedelta(days=no_days - 1)
time_series = []
for i in tqdm(range(no_days), desc=f"Downloading {symbol}"):
current_date = start_date + timedelta(days=i)
try:
trades = download_and_unzip(symbol, current_date, download_dir, cache_dir)
time_series.append(research.timeseries(trades, time_interval, research.OHLC_AGGS))
except Exception as e:
tqdm.write(f"[ERROR] {symbol} {current_date.date()}: {e}")
return pl.concat(time_series)
def download_timeseries(symbol: str, no_days: int, time_interval: str, aggs: List[pl.Expr], download_dir: str = "data", cache_dir: str = "cache") -> pl.DataFrame:
"""
Download trades for the last N days up to yesterday with a progress bar.
"""
yesterday = datetime.now() - timedelta(days=1)
start_date = yesterday - timedelta(days=no_days - 1)
time_series = []
for i in tqdm(range(no_days), desc=f"Downloading {symbol}"):
current_date = start_date + timedelta(days=i)
try:
trades = download_and_unzip(symbol, current_date, download_dir, cache_dir)
time_series.append(research.timeseries(trades, time_interval, aggs))
except Exception as e:
tqdm.write(f"[ERROR] {symbol} {current_date.date()}: {e}")
return pl.concat(time_series)
Generated
-11
View File
@@ -229,15 +229,6 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/1a/39/47f9197bdd44df24d67ac8893641e16f386c984a0619ef2ee4c51fbbc019/beautifulsoup4-4.14.3-py3-none-any.whl", hash = "sha256:0918bfe44902e6ad8d57732ba310582e98da931428d231a5ecb9e7c703a735bb", size = 107721, upload-time = "2025-11-30T15:08:24.087Z" },
]
[[package]]
name = "binance"
version = "0.3.110"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/55/d1/dd29ef0615e0a500657e832c81cc2e3bc9fbc6171fa6106f99cfa3c309fe/binance-0.3.110.tar.gz", hash = "sha256:90a09493cbf64700d78f5257da4ea671e9290068ff7422dfedf21df78ac3836b", size = 939171, upload-time = "2026-06-16T15:14:20.324Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/b6/e2/6ee1ba2e315ff02ea20a95448e3b9d476769577d31667223f535920cc102/binance-0.3.110-py3-none-any.whl", hash = "sha256:f9a7ffbd8d50a8d02057c3fa712e759865c186ca1dfdc7fb4b8016efdf80e109", size = 1201848, upload-time = "2026-06-16T15:14:18.958Z" },
]
[[package]]
name = "bleach"
version = "6.3.0"
@@ -2086,7 +2077,6 @@ source = { virtual = "." }
dependencies = [
{ name = "aiohttp" },
{ name = "altair" },
{ name = "binance" },
{ name = "cryptography" },
{ name = "jupyter" },
{ name = "matplotlib" },
@@ -2103,7 +2093,6 @@ dependencies = [
requires-dist = [
{ name = "aiohttp", specifier = ">=3.14.1" },
{ name = "altair", specifier = ">=6.2.1" },
{ name = "binance", specifier = ">=0.3.110" },
{ name = "cryptography", specifier = ">=49.0.0" },
{ name = "jupyter", specifier = ">=1.1.1" },
{ name = "matplotlib", specifier = ">=3.10.9" },