Support Board

"""
Convert's Text files to SCID files for SierraChart
It will create a new scid file for every text file it finds
format is assumed to be D T O H L C V T with comma separated variables
It will continue to update the scid file every REFRESH seconds
generally refresh would be set to 1/2 the time period for the text file updates
"""

import os
import struct
from time import sleep
from datetime import datetime as dt, timedelta as tdelta
import subprocess

# these are linux format, windows might need to use double backslashes \\
FILES_IN = "/home/john/Python/Trading/textfiles/"
FILES_OUT = "/home/john/zRamdisk/SierraData/"
TIMEZONE = 0 # scid saved as UTC, use this if the text file is another time zone
KEEP_UPDATING = True # set to False to convert and stop; True for live updating
REFRESH = 0.25 # refresh every 1/4 second for 1/2 second updating text files
TEXT_FILE_SUFFIX = ".txt" # usually .txt or .csv

sizeHeader = 0x38
sizeRecord = 0x28
file_header = (b'SCID8\x00\x00\x00\x28\x00\x00\x00\x01\x00\x00\x00\x00\x00' +
b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' +
b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' +
b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00')


def excel_date(datetime, offset=TIMEZONE):
"""convert date&time with offset in hours to excel format for scid"""
delta = datetime - dt(1899, 12, 30) + tdelta(hours=offset)
return delta.days + delta.seconds / 86400


def python_date(exceldt, offset=TIMEZONE):
"""convert date&time with offset in hours to python format"""
excel_as_delta = tdelta(days=exceldt//1) + tdelta(seconds=exceldt%1*86400)
return dt(1899, 12, 30) + excel_as_delta + tdelta(hours=offset)

def write_struct(record, f):
if len(record) == 8 and record[0][0].isnumeric():
out = [excel_date(dt.strptime(f'{record[0]} {record[1]}', '%Y-%m-%d %H:%M:%S.%f'))]
out.extend([float(s) for s in record[2:]])
# note that SC packs dt, O, H, L, C, T, V not V, T so swap them
wt = struct.pack('d4f4I', out[0], out[1], out[2], out[3], out[4], int(out[6]), int(out[5]), 0, 0)
f.write(wt)


def create_base(f):
record = None
with open(FILES_OUT + f[: -3] +'scid', "wb") as scidfile:
scidfile.write(file_header)
with open(FILES_IN + f, "r") as txtfile:
for line in txtfile:
record = line.split(',')
write_struct(record, scidfile)
return f'{record[0]} {record[1]}' # will update when date+time changes


def update_if_changed(files):
for f in files:
with open(FILES_IN + f, "r") as txtfile:
record = txtfile.readlines()[-1].split(',')
last_dt = f'{record[0]} {record[1]}'
if files[f] == last_dt:
continue
files[f] = last_dt
with open(FILES_OUT + f[: -3] +'scid', "ab") as scidfile:
write_struct(record, scidfile)


if __name__ == '__main__':
files = sorted(k for k in os.listdir(path=FILES_IN) if k[-4:] == TEXT_FILE_SUFFIX)
files = {k: '' for k in files}
for f in files:
files[f] = create_base(f)
print('finished base')
if KEEP_UPDATING:
# now test each input file each REFRESH interval & update if datetime changed
while True:
sleep(REFRESH)
update_if_changed(files)