yakenator/yakcloud-starter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

cli v0.10.0: migrate oracle(Data Pump expdp/impdp + REMAP_SCHEMA + EXCLUDE) — 소스 컨테이너 exec+docker cp, chmod-as-root, exclude=user. 9종 전부 지원. docker E2E(캡처·제외·apply-skip·reset복원 CUSTOMERS/ORDERS) 검증

Browse Source
This commit is contained in:
yakenator
2026-08-27 22:21:43 +09:00
parent d8fe2b0ea3
commit 9353cd7706
4 changed files with 76 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

78
scripts/yakcloud_migrate.py
View File

@ -24,11 +24,11 @@ import yakcloud_dev as dev # noqa: E402 (동일 scripts/ 의 dev 엔진 헬퍼
SNAP = "db" # 프로젝트 루트 db/<source>/ (git 추적 — .gitignore 아님)
IGNORE = ".migrateignore"
SUPPORTED = ("postgresql", "mysql", "mariadb", "mongodb", "minio", "redis", "solr", "rabbitmq")
SUPPORTED = ("postgresql", "mysql", "mariadb", "mongodb", "minio", "redis", "solr", "rabbitmq", "oracle")
IMG = {"postgresql": "postgres:16-alpine", "mysql": "mysql:8.4", "mariadb": "mariadb:11.4",
"mongodb": "mongo:7", "minio": "minio/mc:latest",
"redis": "redis:7-alpine", "solr": "solr:9", "rabbitmq": "rabbitmq:3.13-management"}
UNSUPPORTED_NEXT = ("oracle",) # 다음 단계(무거움/설계 보완 필요)
UNSUPPORTED_NEXT = () # 9종 전부 지원(oracle=Data Pump)
def _fill(tmpl: str, **kw) -> str:
@ -170,6 +170,9 @@ def list_units(m: dict, source: str, stype: str, cr: dict) -> list[str]:
return [str(d.get("id", "?")) for d in json.loads(r.stdout or "{}").get("response", {}).get("docs", [])]
except Exception:
return []
elif stype == "oracle":
return _ora_lines(_ora_sql(source, cr,
f"SELECT table_name FROM all_tables WHERE owner = UPPER('{cr['user']}');"))
else:
return []
return [x.strip() for x in (r.stdout or "").splitlines() if x.strip() and "Warning" not in x]
@ -237,6 +240,38 @@ def _solr_capture(net: str, source: str, cr: dict, bl: list[str], outdir: str) -
return {"file": "docs.json", "captured": len(docs)}
# ── oracle: Data Pump(expdp/impdp) — 파일이 DB 서버측이라 소스 컨테이너 exec + docker cp + REMAP_SCHEMA ──
def _ora_sql(source: str, cr: dict, sql: str) -> str:
script = "SET HEADING OFF PAGESIZE 0 FEEDBACK OFF VERIFY OFF ECHO OFF TRIMSPOOL ON\n" + sql + "\nEXIT\n"
r = subprocess.run(["docker", "exec", "-i", f"yakdev-{source}", "bash", "-lc",
f"sqlplus -s system/{cr['sys_pw']}@//localhost:1521/FREEPDB1"],
input=script, text=True, capture_output=True)
return r.stdout or ""
def _ora_lines(out: str) -> list[str]:
bad = ("ORA-", "SP2-", "SQL>", "altered", "PL/SQL", "Connected")
return [ln.strip() for ln in out.splitlines() if ln.strip() and not any(b in ln for b in bad)]
def _ora_dppath(source: str, cr: dict) -> str:
ls = _ora_lines(_ora_sql(source, cr,
"SELECT directory_path FROM dba_directories WHERE directory_name='DATA_PUMP_DIR';"))
return ls[-1] if ls else "/opt/oracle/admin/FREE/dpdump"
def _ora_datapump(source: str, cr: dict, parfile: str, tool: str) -> None:
cont = f"yakdev-{source}"
subprocess.run(["docker", "exec", "-i", cont, "bash", "-lc", "cat > /tmp/yak.par"],
input=parfile, text=True, capture_output=True)
r = subprocess.run(["docker", "exec", cont, "bash", "-lc",
f"{tool} system/{cr['sys_pw']}@//localhost:1521/FREEPDB1 parfile=/tmp/yak.par"],
text=True, capture_output=True)
out = (r.stdout or "") + (r.stderr or "") # expdp/impdp 는 진행상황을 stderr 로 출력
if "successfully completed" not in out:
die(f"{tool} 실패:\n{out[-600:]}")
# ── 미리보기 ────────────────────────────────────────────────────────────
def preview(m: dict, targets: list[tuple[str, str]]) -> dict:
seed = dev.ensure_seed()
@ -255,10 +290,12 @@ def preview(m: dict, targets: list[tuple[str, str]]) -> dict:
f"{os.path.join(_srcdir(source), IGNORE)} 에 Solr 쿼리절(한 줄에 하나)로")
continue
units = list_units(m, source, stype, cr)
inc = [u for u in units if not _excluded(u, bl)]
exc = [u for u in units if _excluded(u, bl)]
blm = [p.upper() for p in bl] if stype == "oracle" else bl # oracle 식별자=대문자
inc = [u for u in units if not _excluded(u, blm)]
exc = [u for u in units if _excluded(u, blm)]
plan[source] = {"type": stype, "include": inc, "exclude": exc, "blacklist": bl}
unit_word = {"minio": "오브젝트", "redis": "키", "rabbitmq": "큐/익스체인지"}.get(stype, "테이블/컬렉션")
unit_word = {"minio": "오브젝트", "redis": "키", "rabbitmq": "큐/익스체인지",
"oracle": "테이블"}.get(stype, "테이블/컬렉션")
print(f"\n ● {source} ({stype}) — {unit_word} {len(units)}개 · 포함 {len(inc)} / 제외 {len(exc)}")
for u in inc[:20]:
print(f" ✓ {u}")
@ -279,6 +316,9 @@ def capture_source(m: dict, source: str, stype: str, cr: dict) -> dict:
bl = _blacklist(source)
if stype == "solr": # solr 는 fq(서버측) 제외라 fnmatch 단위 계산 안 함
units, excluded = [], []
elif stype == "oracle": # oracle 식별자=대문자 → 블랙리스트 글롭도 대문자 정규화 매칭
units = list_units(m, source, stype, cr)
excluded = [u for u in units if _excluded(u, [p.upper() for p in bl])]
else:
units = list_units(m, source, stype, cr)
excluded = [u for u in units if _excluded(u, bl)]
@ -321,6 +361,19 @@ def capture_source(m: dict, source: str, stype: str, cr: dict) -> dict:
info["note"] = "메시지 본문 제외(전이성). 정의(큐/익스체인지/바인딩/정책)만 이관."
elif stype == "solr":
info.update(_solr_capture(net, source, cr, bl, outdir))
elif stype == "oracle":
cont, schema, dp = f"yakdev-{source}", cr["user"].upper(), _ora_dppath(source, cr)
subprocess.run(["docker", "exec", cont, "bash", "-lc", f"rm -f {dp}/yak.dmp"], capture_output=True)
par = f"schemas={schema}\ndirectory=DATA_PUMP_DIR\ndumpfile=yak.dmp\nreuse_dumpfiles=y\nnologfile=y\n"
if excluded:
lst = ", ".join(f"'{e.upper()}'" for e in excluded)
par += f'EXCLUDE=TABLE:"IN ({lst})"\n'
_ora_datapump(source, cr, par, "expdp")
cp = subprocess.run(["docker", "cp", f"{cont}:{dp}/yak.dmp", os.path.join(outdir, "data.dmp")],
capture_output=True, text=True)
if cp.returncode != 0:
die(f"덤프 추출 실패: {cp.stderr[-300:]}")
info["file"], info["schema"] = "data.dmp", schema
else:
die(f"미지원 타입(캡처): {stype}")
json.dump(info, open(os.path.join(outdir, "manifest.json"), "w"), ensure_ascii=False, indent=2)
@ -366,6 +419,21 @@ def apply_source(m: dict, source: str, stype: str, cr: dict) -> str:
drun(net, IMG[stype],
f'curl -sS "http://{source}:8983/solr/{cr["core"]}/update?commit=true" '
f'-H "Content-Type: application/json" --data-binary @/out/docs.json', [(outdir, "/out")])
elif stype == "oracle":
cont, dst, dp = f"yakdev-{source}", cr["user"].upper(), _ora_dppath(source, cr)
src = json.load(open(os.path.join(outdir, "manifest.json"))).get("schema", dst)
cp = subprocess.run(["docker", "cp", os.path.join(outdir, "data.dmp"), f"{cont}:{dp}/yak.dmp"],
capture_output=True, text=True)
if cp.returncode != 0:
die(f"덤프 주입 실패: {cp.stderr[-300:]}")
# docker cp 는 호스트 uid 로 파일 생성 → oracle 프로세스가 읽도록 root(-u 0)로 권한 부여.
subprocess.run(["docker", "exec", "-u", "0", cont, "bash", "-lc", f"chmod 644 {dp}/yak.dmp"],
capture_output=True)
_ora_sql(source, cr, f"ALTER USER {dst} QUOTA UNLIMITED ON USERS;") # 쿼터 보장(멱등)
# exclude=user: 대상 스키마가 이미 존재(gvenzl 생성) → CREATE USER 스킵(ORA-31684 benign 방지, 깨끗한 성공)
par = (f"directory=DATA_PUMP_DIR\ndumpfile=yak.dmp\nnologfile=y\nexclude=user\n"
f"remap_schema={src}:{dst}\ntable_exists_action=skip\n")
_ora_datapump(source, cr, par, "impdp")
else:
die(f"미지원 타입(적용): {stype}")
return "적용 완료(부트스트랩)"