Skip to content

Commit

Permalink
fixing schema name management
Browse files Browse the repository at this point in the history
  • Loading branch information
scicco committed Nov 19, 2024
1 parent 1e9be6a commit 05c25b6
Show file tree
Hide file tree
Showing 2 changed files with 11 additions and 5 deletions.
6 changes: 3 additions & 3 deletions load_mysql.py
Original file line number Diff line number Diff line change
Expand Up @@ -280,10 +280,10 @@ def process_csv(csv, cdm_schema, vocab_file_dir, chunk_size=1000000):

conn = engine.connect()

if cdm_schema is not None:
table_name = f"{cdm_schema}.{csv.split('.')[0]}"
if cdm_schema != '':
table_name = f"\"{cdm_schema}\".{csv.split('.')[0]}"
else:
table_name = csv.split('.')[0]
table_name = f"{csv.split('.')[0]}"

table_name = table_name.lower()
cursor = conn.connection.cursor()
Expand Down
10 changes: 8 additions & 2 deletions load_vocab.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,11 @@ def process_csv(csv, connection_details, cdm_schema, vocab_file_dir, chunk_size=
port=connection_details["port"]
)

table_name = f"{cdm_schema}.{csv.split('.')[0]}"
if cdm_schema != '':
table_name = f"\"{cdm_schema}\".{csv.split('.')[0]}"
else:
table_name = f"{csv.split('.')[0]}"

with conn.cursor() as cur:
cur.execute(f"DELETE FROM {table_name};")

Expand All @@ -50,8 +54,10 @@ def process_csv(csv, connection_details, cdm_schema, vocab_file_dir, chunk_size=
tuples = [tuple(x) for x in chunk.to_numpy()]
cols = ','.join(list(chunk.columns))
query = f"INSERT INTO {table_name}({cols}) VALUES %s"

psycopg2.extras.execute_values(cur, query, tuples, template=None, page_size=1000)



processed_lines += len(chunk)
print(f"Processed lines: {processed_lines}, Remaining lines: {total_lines - processed_lines}")

Expand Down

0 comments on commit 05c25b6

Please sign in to comment.