Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 2dbd569

Browse files
Rajandeep98Rajandeep98
authored
34037 - Refresh Data Dev (#4554)
* 34037 - Refresh Data Dev * Added Oracle config * updated * updated * updated * updated build changes * updated * added oracle driver * added config * updated config * added driver * updated driver extraction * updated driver removal * updated docker file * updated * updated * added config * updated * updated
1 parent d216916 commit 2dbd569

5 files changed

Lines changed: 34 additions & 15 deletions

File tree

‎jobs/colin-extract-refresh/Dockerfile‎

Lines changed: 18 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ ENV DBSCHEMA_VERSION=9_7_1
4040
RUN set -eux; \
4141
apt-get update; \
4242
apt-get install -y --no-install-recommends default-jre-headless curl;\
43-
curl -fsSL "https://www.dbschema.com/download/dbschema_unix_9_7_1.tar.gz" -o /tmp/dbschema.tar.gz;\
43+
curl --http1.1 -fsSL "https://www.dbschema.com/download/dbschema_unix_9_7_1.tar.gz" -o /tmp/dbschema.tar.gz;\
4444
tar -xzf /tmp/dbschema.tar.gz -C /opt; \
4545
rm /tmp/dbschema.tar.gz; \
4646
mkdir -p "${DBSCHEMA_HOME}/lib"; \
@@ -50,9 +50,21 @@ RUN set -eux; \
5050
rm -rf /var/lib/apt/lists/*
5151

5252
ENV HOME=/opt/app-root
53-
RUN mkdir -p /opt/app-root/.DbSchema/cli /opt/app-root/.DbSchema/drivers/PostgreSql && \
54-
curl -fsSL "https://repo1.maven.org/maven2/org/postgresql/postgresql/42.7.4/postgresql-42.7.4.jar" -o /opt/app-root/.DbSchema/drivers/PostgreSql/postgresql.jar && \
55-
chmod -R g+rwX /opt/app-root/.DbSchema && \
53+
54+
RUN mkdir -p ${HOME}/.DbSchema/cli ${HOME}/.DbSchema/drivers/PostgreSql ${HOME}/.DbSchema/drivers/Oracle && \
55+
curl -fsSL "https://repo1.maven.org/maven2/org/postgresql/postgresql/42.7.4/postgresql-42.7.4.jar" -o ${HOME}/.DbSchema/drivers/PostgreSql/postgresql.jar && \
56+
printf '%s\n' 'download driver oracle' > /tmp/download_oracle_driver.sql && \
57+
dbschemacli /tmp/download_oracle_driver.sql && \
58+
if [ -f /root/.DbSchema/drivers/oracle/driver.zip ]; then \
59+
unzip -o /root/.DbSchema/drivers/oracle/driver.zip -d ${HOME}/.DbSchema/drivers/Oracle/; \
60+
elif [ -f ${HOME}/.DbSchema/drivers/oracle/driver.zip ]; then \
61+
unzip -o ${HOME}/.DbSchema/drivers/oracle/driver.zip -d ${HOME}/.DbSchema/drivers/Oracle/; \
62+
else \
63+
echo "Oracle JDBC driver download failed" >&2; exit 1; \
64+
fi && \
65+
rm -rf /root/.DbSchema/drivers/oracle ${HOME}/.DbSchema/drivers/oracle/driver.zip
66+
67+
RUN chmod -R g+rwX ${HOME}/.DbSchema && \
5668
chmod 755 /opt/app-root
5769

5870
WORKDIR /opt/app-root
@@ -64,7 +76,7 @@ RUN apt-get update; \
6476

6577
ENV PATH="${DBSCHEMA_HOME}:${PATH}"
6678

67-
COPY src/ ./colin-exrtact-refresh/src/
79+
COPY src/ ./colin-extract-refresh/src/
6880
RUN mkdir -p /opt/app-root/colin-extract-refresh/src/subset/generated && \
6981
chmod -R 777 /opt/app-root/colin-extract-refresh/src/subset/generated /opt/app-root/.DbSchema
7082

@@ -74,5 +86,4 @@ EXPOSE 8080
7486

7587
COPY src .
7688

77-
ENTRYPOINT ["python", "generate_cprd_subset_extract.py"]
78-
CMD ["--corp-file", "corp.txt", "--mode", "refresh"]
89+
CMD ["/bin/sh", "-c", "python /opt/app-root/colin-extract-refresh/src/test_connectivity.py && python /opt/app-root/colin-extract-refresh/src/generate_cprd_subset_extract.py --corp-file corp.txt --target-connection my_proxy_test --target-schema colin_extract_temp --mode refresh && dbschemacli /opt/app-root/colin-extract-refresh/src/subset/generated/subset_refresh.sql"]

‎jobs/colin-extract-refresh/src/corp.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
BC0009721

‎jobs/colin-extract-refresh/src/dbschemacli_init.py‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,11 +11,18 @@ def write_dbschema_init(cfg: _Config) -> Path:
1111
host = cfg.DB_HOST_COLIN_MIGR
1212
port = int(cfg.DB_PORT_COLIN_MIGR)
1313
name = cfg.DB_NAME_COLIN_MIGR
14+
15+
ora_user = cfg.DB_USER_COLIN_ORACLE
16+
ora_password = cfg.DB_PASSWORD_COLIN_ORACLE
17+
ora_host = cfg.DB_HOST_COLIN_ORACLE
18+
ora_port = int(cfg.DB_PORT_COLIN_ORACLE)
19+
ora_name = cfg.DB_NAME_COLIN_ORACLE
1420

1521
lines = [
1622
'register driver PostgreSql org.postgresql.Driver jdbc:postgresql://<host>:<port>/<db>?reWriteBatchedInserts=true "port=5432"',
23+
'register driver Oracle oracle.jdbc.OracleDriver jdbc:oracle:thin:@<host>:<port>:<db> "port=1521"',
1724
f'connection my_proxy_test -d PostgreSql -u {user} -p {password} -h {host} -P {port} -D {name}',
18-
25+
f'connection cprd -d Oracle -u {ora_user} -p {ora_password} -h {ora_host} -P {ora_port} -D {ora_name}',
1926
]
2027

2128
init_path.write_text('\n'.join(lines) + '\n')

‎jobs/colin-extract-refresh/src/generate_cprd_subset_extract.py‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -592,9 +592,9 @@ def _gen_emit_pg_disable_begin(lines: List[str], *, cfg: cfg_GenerationConfig, t
592592
lines.append(f"execute {tmpl_resolve_execute_path(templates.disable_triggers, out_dir=cfg.out_chunks_dir).as_posix()}")
593593
if cfg.mode == cfg_GenerationMode.REFRESH:
594594
lines.append("-- Refresh-only: preserved processing/tracking tables still reference corporation/event rows.")
595-
lines.append("ALTER TABLE corp_processing DISABLE TRIGGER ALL;")
596-
# lines.append("ALTER TABLE auth_processing DISABLE TRIGGER ALL;")
597-
lines.append("ALTER TABLE colin_tracking DISABLE TRIGGER ALL;")
595+
# lines.append("ALTER TABLE corp_processing DISABLE TRIGGER ALL;")
596+
# # lines.append("ALTER TABLE auth_processing DISABLE TRIGGER ALL;")
597+
# lines.append("ALTER TABLE colin_tracking DISABLE TRIGGER ALL;")
598598
lines.append("")
599599
return
600600

@@ -612,9 +612,9 @@ def _gen_emit_pg_disable_end(lines: List[str], *, cfg: cfg_GenerationConfig, tem
612612
lines.append(f"execute {tmpl_resolve_execute_path(templates.enable_triggers, out_dir=cfg.out_chunks_dir).as_posix()}")
613613
if cfg.mode == cfg_GenerationMode.REFRESH:
614614
lines.append("-- Refresh-only: restore preserved processing/tracking table triggers too.")
615-
lines.append("ALTER TABLE corp_processing ENABLE TRIGGER ALL;")
616-
# lines.append("ALTER TABLE auth_processing ENABLE TRIGGER ALL;")
617-
lines.append("ALTER TABLE colin_tracking ENABLE TRIGGER ALL;")
615+
# lines.append("ALTER TABLE corp_processing ENABLE TRIGGER ALL;")
616+
# # lines.append("ALTER TABLE auth_processing ENABLE TRIGGER ALL;")
617+
# lines.append("ALTER TABLE colin_tracking ENABLE TRIGGER ALL;")
618618
lines.append("")
619619
return
620620

‎jobs/colin-extract-refresh/src/test_connectivity.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
def main() -> int:
77
print("== running test_connectivity.py ==")
88
run_business_check()
9-
# run_colin_check()
9+
run_colin_check()
1010
print("== done test_connectivity.py ==")
1111
return 0
1212

0 commit comments

Comments
 (0)