From 54e1e72b94a5ef5fc5491ad64a92d182594f62bb Mon Sep 17 00:00:00 2001 From: Xiao-QvQ Date: Wed, 8 Jul 2026 12:35:14 +0800 Subject: [PATCH] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E4=B8=AD=E6=96=87=E6=B3=A8?= =?UTF-8?q?=E9=87=8A=EF=BC=8C=E5=A2=9E=E5=BC=BA=E5=8F=AF=E8=AF=BB=E6=80=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .vscode/launch.json | 16 +++++++ python/download-kline.py | 68 ++++++++++++++++++++++------ python/enums.py | 20 ++++---- python/utility.py | 98 +++++++++++++++++++++++++++------------- 4 files changed, 147 insertions(+), 55 deletions(-) create mode 100644 .vscode/launch.json diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..3b85f8c --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,16 @@ +{ + // 使用 IntelliSense 了解相关属性。 + // 悬停以查看现有属性的描述。 + // 欲了解更多信息，请访问: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Python 调试程序: 包含参数的当前文件", + "type": "debugpy", + "request": "launch", + "program": "python/download-kline.py", + "console": "integratedTerminal", + "args": ["-s", "BNBUSDT", "-t", "spot", "-i", "1h", "-startDate", "2018-12-15", "-c", "1", "-skip-daily", "1"] + } + ] +} \ No newline at end of file diff --git a/python/download-kline.py b/python/download-kline.py index 1be6791..07f40fd 100755 --- a/python/download-kline.py +++ b/python/download-kline.py @@ -1,12 +1,10 @@ #!/usr/bin/env python +''' +用于下载 K 线数据的脚本。 +请为存储目录（STORE_DIRECTORY）设置目标文件夹的绝对路径，随后执行脚本。 +示例：STORE_DIRECTORY=/data/ ./download-kline.py +''' -""" - script to download klines. - set the absolute path destination folder for STORE_DIRECTORY, and run - - e.g. STORE_DIRECTORY=/data/ ./download-kline.py - -""" import sys from datetime import * import pandas as pd @@ -16,6 +14,20 @@ def download_monthly_klines(trading_type, symbols, num_symbols, intervals, years, months, start_date, end_date, folder, checksum): + ''' + 下载月 K 线 + trading_type: 交易类型（现货、美元本位合约、币本位合约） + symbols: 币种列表 + num_symbols: 币种数量 + intervals: K 周期 + years: 支持的年份列表 + months: 月份列表 + start_date: 起始日期 + end_date: 结束日期 + folder: 存放下载数据的目录 + checksum: 是否下载校验文件 + ''' + #当前数量 current = 0 date_range = None @@ -26,23 +38,27 @@ def download_monthly_klines(trading_type, symbols, num_symbols, intervals, years start_date = START_DATE else: start_date = convert_to_date_object(start_date) + start_date = start_date.replace(day=1) if not end_date: end_date = END_DATE else: end_date = convert_to_date_object(end_date) + end_date = end_date.replace(day=1) - print("Found {} symbols".format(num_symbols)) + print("币种列表数量：{}".format(num_symbols)) for symbol in symbols: - print("[{}/{}] - start download monthly {} klines ".format(current+1, num_symbols, symbol)) + print("[{}/{}] - 开始下载月度 [{}] K线数据".format(current+1, num_symbols, symbol)) for interval in intervals: for year in years: for month in months: current_date = convert_to_date_object('{}-{}-01'.format(year, month)) + + #判断当前日期是否在起始日期和结束日期之间 if current_date >= start_date and current_date <= end_date: path = get_path(trading_type, "klines", "monthly", symbol, interval) - file_name = "{}-{}-{}-{}.zip".format(symbol.upper(), interval, year, '{:02d}'.format(month)) + file_name = "{}-{}-{}-{}.zip".format(symbol.upper(), interval, year, '{:02d}'.format(month))#获取网络上的文件名称 download_file(path, file_name, date_range, folder) if checksum == 1: @@ -53,6 +69,18 @@ def download_monthly_klines(trading_type, symbols, num_symbols, intervals, years current += 1 def download_daily_klines(trading_type, symbols, num_symbols, intervals, dates, start_date, end_date, folder, checksum): + ''' + 下载日 K 线 + trading_type: 交易类型（现货、美元本位合约、币本位合约） + symbols: 币种列表 + num_symbols: 币种数量 + intervals: K 线时间间隔列表 + dates: 日期列表 + start_date: 起始日期 + end_date: 结束日期 + folder: 存放下载数据的目录 + checksum: 是否下载校验文件 + ''' current = 0 date_range = None @@ -91,26 +119,36 @@ def download_daily_klines(trading_type, symbols, num_symbols, intervals, dates, current += 1 if __name__ == "__main__": + print(sys.argv) parser = get_parser('klines') args = parser.parse_args(sys.argv[1:]) + #获取币种列表和数量 if not args.symbols: - print("fetching all symbols from exchange") + #获取交易所所有币种 + print("从交易所获取全部交易标的") symbols = get_all_symbols(args.type) num_symbols = len(symbols) else: + #获取币种列表 symbols = args.symbols + #获取币种数量 num_symbols = len(symbols) + #获取日期列表 if args.dates: dates = args.dates else: - period = convert_to_date_object(datetime.today().strftime('%Y-%m-%d')) - convert_to_date_object( - PERIOD_START_DATE) + #设置默认起始日期和结束日期 + period = convert_to_date_object(datetime.today().strftime('%Y-%m-%d')) -\ + convert_to_date_object(args.startDate if args.startDate is not None else PERIOD_START_DATE) dates = pd.date_range(end=datetime.today(), periods=period.days + 1).to_pydatetime().tolist() dates = [date.strftime("%Y-%m-%d") for date in dates] - if args.skip_monthly == 0: - download_monthly_klines(args.type, symbols, num_symbols, args.intervals, args.years, args.months, args.startDate, args.endDate, args.folder, args.checksum) + + #是否下载月度 K 线数据 + if args.skip_monthly == 0: + download_monthly_klines(args.type, symbols, num_symbols, args.intervals, args.years, args.months, args.startDate, args.endDate, args.folder, args.checksum) + #是否下载日 K 线数据 if args.skip_daily == 0: download_daily_klines(args.type, symbols, num_symbols, args.intervals, dates, args.startDate, args.endDate, args.folder, args.checksum) diff --git a/python/enums.py b/python/enums.py index ae612a2..02f7196 100644 --- a/python/enums.py +++ b/python/enums.py @@ -1,11 +1,13 @@ from datetime import * -YEARS = ['2017', '2018', '2019', '2020', '2021', '2022', '2023', '2024', '2025'] -INTERVALS = ["1s", "1m", "3m", "5m", "15m", "30m", "1h", "2h", "4h", "6h", "8h", "12h", "1d", "3d", "1w", "1mo"] -DAILY_INTERVALS = ["1s", "1m", "3m", "5m", "15m", "30m", "1h", "2h", "4h", "6h", "8h", "12h", "1d"] -TRADING_TYPE = ["spot", "um", "cm"] -MONTHS = list(range(1,13)) -PERIOD_START_DATE = '2020-01-01' -BASE_URL = 'https://data.binance.vision/' -START_DATE = date(int(YEARS[0]), MONTHS[0], 1) -END_DATE = datetime.date(datetime.now()) +YEARS = ['2017', '2018', '2019', '2020', '2021', '2022', '2023', '2024', '2025', '2026']#年份定义 +INTERVALS = ["1s", "1m", "3m", "5m", "15m", "30m", "1h", "2h", "4h", "6h", "8h", "12h", "1d", "3d", "1w", "1mo"]#周期格式 +DAILY_INTERVALS = ["1s", "1m", "3m", "5m", "15m", "30m", "1h", "2h", "4h", "6h", "8h", "12h", "1d"]#下载支持的周期 +TRADING_TYPE = ["spot", "um", "cm"]#数据类型：现货、美元本位合约、币本位合约 +MONTHS = list(range(1,13))#月份定义 +PERIOD_START_DATE = '2020-01-01'#默认起始日期 +BASE_URL = 'https://data.binance.vision/'#数据网址 + +START_DATE = date(int(YEARS[0]), MONTHS[0], 1)#默认开始日期 +END_DATE = datetime.date(datetime.now())#默认结束日期 +SAVE_RANGE_SEPARATE_FOLDER = False#是否将不同日期范围的数据保存到不同的文件夹中 \ No newline at end of file diff --git a/python/utility.py b/python/utility.py index 72d3ee6..cfad3e6 100644 --- a/python/utility.py +++ b/python/utility.py @@ -7,6 +7,11 @@ from enums import * def get_destination_dir(file_url, folder=None): + ''' + 获取文件的保存路径 + file_url: 文件在网络上的路径 + folder: 保存文件的目录 + ''' store_directory = os.environ.get('STORE_DIRECTORY') if folder: store_directory = folder @@ -15,9 +20,17 @@ def get_destination_dir(file_url, folder=None): return os.path.join(store_directory, file_url) def get_download_url(file_url): + ''' + 获取文件的下载路径 + file_url: 文件在网络上的路径 + ''' return "{}{}".format(BASE_URL, file_url) def get_all_symbols(type): + ''' + 获取所有交易币种 + type: 交易类型（现货、美元本位合约、币本位合约） + ''' if type == 'um': response = urllib.request.urlopen("https://fapi.binance.com/fapi/v1/exchangeInfo").read() elif type == 'cm': @@ -27,70 +40,85 @@ def get_all_symbols(type): return list(map(lambda symbol: symbol['symbol'], json.loads(response)['symbols'])) def download_file(base_path, file_name, date_range=None, folder=None): + ''' + 从网络下载文件 + base_path: 文件在网络上的路径 + file_name: 文件名称 + date_range: 日期范围 + folder: 保存文件的目录 + ''' + + #获取文件的保存路径 save_path download_path = "{}{}".format(base_path, file_name) if folder: base_path = os.path.join(folder, base_path) - if date_range: - date_range = date_range.replace(" ","_") - base_path = os.path.join(base_path, date_range) + if date_range and SAVE_RANGE_SEPARATE_FOLDER: + date_range = date_range.replace(" ","_") + base_path = os.path.join(base_path, date_range) save_path = get_destination_dir(os.path.join(base_path, file_name), folder) - + #检查文件是否存在 if os.path.exists(save_path): - print("\nfile already exists! {}".format(save_path)) + print("\n文件已存在！ {}".format(save_path)) return - # make the directory + # 创建目录 if not os.path.exists(base_path): Path(get_destination_dir(base_path)).mkdir(parents=True, exist_ok=True) + #正式开始下载文件 try: download_url = get_download_url(download_path) - dl_file = urllib.request.urlopen(download_url) - length = dl_file.getheader('content-length') + dl_file = urllib.request.urlopen(download_url)#连接到网络文件 + length = dl_file.getheader('content-length')#获取文件大小（以字节为单位） if length: length = int(length) - blocksize = max(4096,length//100) + blocksize = max(4096,length//100)#设置缓冲区大小 - with open(save_path, 'wb') as out_file: - dl_progress = 0 - print("\nFile Download: {}".format(save_path)) + with open(save_path, 'wb') as out_file:#以二进制写入模式打开文件 + dl_progress = 0#下载进度 + print("\n下载文件: {}".format(save_path)) while True: - buf = dl_file.read(blocksize) + buf = dl_file.read(blocksize)#读取数据到缓冲区 if not buf: break dl_progress += len(buf) out_file.write(buf) + #显示下载进度 done = int(50 * dl_progress / length) sys.stdout.write("\r[%s%s]" % ('#' * done, '.' * (50-done)) ) sys.stdout.flush() except urllib.error.HTTPError: - print("\nFile not found: {}".format(download_url)) + print("\n未找到文件: {}".format(download_url)) pass def convert_to_date_object(d): + #将字符串转换为日期对象 year, month, day = [int(x) for x in d.split('-')] date_obj = date(year, month, day) return date_obj def get_start_end_date_objects(date_range): + start, end = date_range.split() start_date = convert_to_date_object(start) end_date = convert_to_date_object(end) return start_date, end_date def match_date_regex(arg_value, pat=re.compile(r'\d{4}-\d{2}-\d{2}')): + #检查日期格式是否正确 if not pat.match(arg_value): raise ArgumentTypeError return arg_value def check_directory(arg_value): + #检查文件夹是否存在 if os.path.exists(arg_value): while True: - option = input('Folder already exists! Do you want to overwrite it? y/n ') + option = input('文件夹已存在！是否覆盖？是 / 否') if option != 'y' and option != 'n': - print('Invalid Option!') + print('无效选项！') continue elif option == 'y': shutil.rmtree(arg_value) @@ -100,9 +128,18 @@ def check_directory(arg_value): return arg_value def raise_arg_error(msg): + #抛出参数错误异常 raise ArgumentTypeError(msg) def get_path(trading_type, market_data_type, time_period, symbol, interval=None): + ''' + 获取网络上文件路径 + trading_type: 交易类型（现货、美元本位合约、币本位合约） + market_data_type: 市场数据类型（K线、交易、深度等） + time_period: 保存数据的时间周期（月度、日线） + symbol: 币种（BTCUSDT、ETHUSDT等） + interval: K线周期（1m、1d等） + ''' trading_type_path = 'data/spot' if trading_type != 'spot': trading_type_path = f'data/futures/{trading_type}' @@ -113,47 +150,46 @@ def get_path(trading_type, market_data_type, time_period, symbol, interval=None) return path def get_parser(parser_type): - parser = ArgumentParser(description=("This is a script to download historical {} data").format(parser_type), formatter_class=RawTextHelpFormatter) + #命令行参数解析器 + parser = ArgumentParser(description=("这是一段用于下载历史 {} 数据的脚本").format(parser_type), formatter_class=RawTextHelpFormatter) parser.add_argument( '-s', dest='symbols', nargs='+', - help='Single symbol or multiple symbols separated by space') + help='单个符号，或由空格分隔的多个符号') parser.add_argument( '-y', dest='years', default=YEARS, nargs='+', choices=YEARS, - help='Single year or multiple years separated by space\n-y 2019 2021 means to download {} from 2019 and 2021'.format(parser_type)) + help='单个年份或多个以空格分隔的年份。\n -y 2019 2021 表示从 2019 年和 2021 年下载 {} 。'.format(parser_type)) parser.add_argument( '-m', dest='months', default=MONTHS, nargs='+', type=int, choices=MONTHS, - help='Single month or multiple months separated by space\n-m 2 12 means to download {} from feb and dec'.format(parser_type)) + help='单个月份或多个以空格分隔的月份。\n -m 2 12 表示从 2 月和 12 月下载 {} 。'.format(parser_type)) parser.add_argument( '-d', dest='dates', nargs='+', type=match_date_regex, - help='Date to download in [YYYY-MM-DD] format\nsingle date or multiple dates separated by space\ndownload from 2020-01-01 if no argument is parsed') + help='需下载的日期，格式为 [YYYY-MM-DD]。\n可输入单个日期或多个以空格分隔的日期。\n若未解析到参数，则从 {}开始下载。'.format(datetime.strptime(PERIOD_START_DATE, '%Y-%m-%d').strftime('%Y年%m月%d日'))) parser.add_argument( '-startDate', dest='startDate', type=match_date_regex, - help='Starting date to download in [YYYY-MM-DD] format') + help='下载起始日期（格式：年-月-日）') parser.add_argument( '-endDate', dest='endDate', type=match_date_regex, - help='Ending date to download in [YYYY-MM-DD] format') + help='下载结束日期（格式：年-月-日）') parser.add_argument( '-folder', dest='folder', type=check_directory, - help='Directory to store the downloaded data') + help='存放下载数据的目录') parser.add_argument( '-skip-monthly', dest='skip_monthly', default=0, type=int, choices=[0, 1], - help='1 to skip downloading of monthly data, default 0') + help='输入 1 则跳过月度数据下载，默认值为 0') parser.add_argument( '-skip-daily', dest='skip_daily', default=0, type=int, choices=[0, 1], - help='1 to skip downloading of daily data, default 0') + help='输入 1 则跳过日线数据下载，默认值为 0') parser.add_argument( '-c', dest='checksum', default=0, type=int, choices=[0,1], - help='1 to download checksum file, default 0') + help='输入 1 则下载校验文件，默认值为 0') parser.add_argument( '-t', dest='type', required=True, choices=TRADING_TYPE, - help='Valid trading types: {}'.format(TRADING_TYPE)) + help='有效的交易类型: {}'.format(TRADING_TYPE)) if parser_type == 'klines': parser.add_argument( '-i', dest='intervals', default=INTERVALS, nargs='+', choices=INTERVALS, - help='single kline interval or multiple intervals separated by space\n-i 1m 1w means to download klines interval of 1minute and 1week') - - + help='单个 K 线时间间隔或多个以空格分隔的时间间隔。\n -i 1m 1w 表示下载 1 分钟和 1 周时间间隔的 K 线数据。') return parser