mirror of
https://github.com/maziggy/bambuddy.git
synced 2026-08-11 00:30:12 -04:00
158301ac8a
The three remaining review items, all in the read path. E1 — four provider calls where two would do. preview() called list_commits twice: once inside _resolve_ref to turn HEAD into a SHA, once more at limit=20 purely to find the entry describing that same SHA. And list_tree's recursive tree GET was thrown away, so fetch_files immediately fetched the identical tree again to map path -> blob SHA. _resolve_ref now returns the entry it already has, and list_tree returns its blob_shas map for fetch_files to take as an optional argument. GitLab reads files by path and ignores it. E2 — `commit: null` for a ref outside the 20 most recent. Two causes, and the second is the one that actually bit: REF_PATTERN accepts a 7-character ref while providers return the full 40, so the exact `==` in the scan never matched an abbreviated SHA *even when the commit was in the window*. Fixed by prefix comparison, plus a get_commit(ref) on the GitHub and GitLab backends for the genuinely-outside-the-window case. Gitea and Forgejo inherit GitHub's. Still best-effort: it is a subject line and a date, so a failed lookup renders the preview without them rather than failing it. E7 — the two tree readers disagreed, and each was wrong in the other's direction. GitHub's recursive trees endpoint is not paginated and signals overflow with truncated=true, which _blob_shas_at hard-fails on. Gitea and Forgejo *do* page that endpoint, and inherited that single GET unchanged — so a large backup repo returned only the first page and every category beyond it looked absent from the commit. GiteaBackend now has its own paging _blob_shas_at. GitLab had the mirror-image bug the review did not name: at its 50-page cap it exited through the while condition and returned success: True with a silently partial path list. Both now fail loudly, which is what the GitHub version was always doing. Both halves of E7 are the same failure the module already refuses to allow: a restore that skips categories and calls it "not present in this backup commit". 24 new or changed tests, all failing against this commit's parent.
527 lines
23 KiB
Python
527 lines
23 KiB
Python
"""GitLab backend — implements GitProviderBackend using the GitLab REST API v4."""
|
|
|
|
import base64
|
|
import json
|
|
import logging
|
|
import re
|
|
import urllib.parse
|
|
from datetime import datetime, timezone
|
|
|
|
import httpx
|
|
|
|
from backend.app.services.git_providers.base import GitProviderBackend
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class GitLabBackend(GitProviderBackend):
|
|
"""Backend for gitlab.com and self-hosted GitLab instances."""
|
|
|
|
def get_api_base(self, repo_url: str) -> str:
|
|
match = re.match(r"(https?://[\w.\-]+(:\d+)?)/", repo_url)
|
|
if not match:
|
|
raise ValueError(f"Cannot derive API base from URL: {repo_url}")
|
|
return f"{match.group(1)}/api/v4"
|
|
|
|
def get_headers(self, token: str) -> dict:
|
|
return {"Authorization": f"Bearer {token}", "Content-Type": "application/json"}
|
|
|
|
def parse_repo_url(self, url: str) -> tuple[str, str]:
|
|
"""Return (namespace, repo) from HTTPS or SSH URL.
|
|
|
|
namespace may include subgroups, e.g. 'group/subgroup' for
|
|
gitlab.com/group/subgroup/project. Callers join them with '/' and
|
|
URL-encode the result for /api/v4/projects/{encoded_path}.
|
|
"""
|
|
if not url or len(url) > 500:
|
|
raise ValueError("Invalid Git URL: URL too long or empty")
|
|
match = re.match(r"https?://[\w.\-]+(:\d+)?/(.+?)(?:\.git)?/?$", url)
|
|
if match:
|
|
full_path = match.group(2)
|
|
if "/" not in full_path:
|
|
raise ValueError(f"Cannot parse repository URL: {url}")
|
|
namespace, _, repo = full_path.rpartition("/")
|
|
return namespace, repo
|
|
match = re.match(r"git@[\w.\-]+:(.+?)(?:\.git)?$", url)
|
|
if match:
|
|
full_path = match.group(1)
|
|
if "/" not in full_path:
|
|
raise ValueError(f"Cannot parse repository URL: {url}")
|
|
namespace, _, repo = full_path.rpartition("/")
|
|
return namespace, repo
|
|
raise ValueError(f"Cannot parse repository URL: {url}")
|
|
|
|
async def test_connection(self, repo_url: str, token: str, client: httpx.AsyncClient) -> dict:
|
|
try:
|
|
owner, repo = self.parse_repo_url(repo_url)
|
|
api_base = self.get_api_base(repo_url)
|
|
headers = self.get_headers(token)
|
|
encoded_path = urllib.parse.quote(f"{owner}/{repo}", safe="")
|
|
|
|
response = await client.get(f"{api_base}/projects/{encoded_path}", headers=headers)
|
|
|
|
if response.status_code == 401:
|
|
return {"success": False, "message": "Invalid access token", "repo_name": None, "permissions": None}
|
|
if response.status_code == 404:
|
|
return {
|
|
"success": False,
|
|
"message": "Repository not found. Check URL and token permissions.",
|
|
"repo_name": None,
|
|
"permissions": None,
|
|
}
|
|
if response.status_code != 200:
|
|
return {
|
|
"success": False,
|
|
"message": f"API error: {response.status_code}",
|
|
"repo_name": None,
|
|
"permissions": None,
|
|
}
|
|
|
|
data = response.json()
|
|
perms = data.get("permissions") or {}
|
|
project_level = (perms.get("project_access") or {}).get("access_level", 0)
|
|
group_level = (perms.get("group_access") or {}).get("access_level", 0)
|
|
effective = max(project_level, group_level)
|
|
|
|
# GitLab uses visibility="private" / "internal" / "public". Both
|
|
# "internal" (signed-in users) and "public" are non-private for
|
|
# the purposes of this safety check.
|
|
visibility = (data.get("visibility") or "").lower()
|
|
is_private = visibility == "private"
|
|
|
|
if effective < 30: # Developer = 30, Maintainer = 40, Owner = 50
|
|
return {
|
|
"success": False,
|
|
"message": "Token requires Developer access or higher to push",
|
|
"repo_name": data.get("name_with_namespace"),
|
|
"permissions": perms,
|
|
"is_private": is_private,
|
|
}
|
|
|
|
return {
|
|
"success": True,
|
|
"message": "Connection successful",
|
|
"repo_name": data.get("name_with_namespace"),
|
|
"permissions": perms,
|
|
"is_private": is_private,
|
|
}
|
|
except Exception as e:
|
|
logger.error("GitLab connection test failed: %s", e)
|
|
return {
|
|
"success": False,
|
|
"message": f"Connection failed: {type(e).__name__}",
|
|
"repo_name": None,
|
|
"permissions": None,
|
|
"is_private": None,
|
|
}
|
|
|
|
def _encoded_project(self, repo_url: str) -> str:
|
|
"""Return the URL-encoded ``namespace/project`` path for /api/v4/projects/."""
|
|
owner, repo = self.parse_repo_url(repo_url)
|
|
return urllib.parse.quote(f"{owner}/{repo}", safe="")
|
|
|
|
async def list_commits(
|
|
self,
|
|
repo_url: str,
|
|
token: str,
|
|
branch: str,
|
|
client: httpx.AsyncClient,
|
|
limit: int = 20,
|
|
) -> dict:
|
|
"""List recent commits on ``branch`` via /repository/commits."""
|
|
try:
|
|
api_base = self.get_api_base(repo_url)
|
|
headers = self.get_headers(token)
|
|
encoded_path = self._encoded_project(repo_url)
|
|
|
|
response = await client.get(
|
|
f"{api_base}/projects/{encoded_path}/repository/commits",
|
|
headers=headers,
|
|
params={"ref_name": branch, "per_page": limit},
|
|
)
|
|
|
|
if response.status_code == 404:
|
|
return {
|
|
"success": False,
|
|
"message": (
|
|
f"Branch '{branch}' not found, or the repository has no commits yet. "
|
|
"Run a backup before restoring."
|
|
),
|
|
"commits": [],
|
|
}
|
|
if response.status_code != 200:
|
|
msg = f"Failed to list commits (HTTP {response.status_code}): {self._truncated_response_text(response)}"
|
|
logger.warning("list_commits %s: %s", repo_url, msg)
|
|
return {"success": False, "message": msg, "commits": []}
|
|
|
|
try:
|
|
data = response.json()
|
|
except ValueError:
|
|
return {"success": False, "message": "Non-JSON response listing commits", "commits": []}
|
|
if not isinstance(data, list):
|
|
return {"success": False, "message": "Unexpected shape listing commits", "commits": []}
|
|
|
|
commits = []
|
|
for entry in data[:limit]:
|
|
if not isinstance(entry, dict):
|
|
continue
|
|
sha = entry.get("id")
|
|
if not isinstance(sha, str) or not sha:
|
|
continue
|
|
# GitLab flattens author/date onto the commit itself rather than
|
|
# nesting them under "commit" the way GitHub does.
|
|
commits.append(
|
|
{
|
|
"sha": sha,
|
|
"message": entry.get("message") or "",
|
|
"author": entry.get("author_name") or "",
|
|
"date": entry.get("committed_date") or entry.get("created_at") or "",
|
|
}
|
|
)
|
|
|
|
return {"success": True, "message": "OK", "commits": commits}
|
|
|
|
except Exception as e:
|
|
logger.exception("list_commits failed for %s branch=%s", repo_url, branch)
|
|
return {"success": False, "message": f"{type(e).__name__}: {str(e)[:200]}", "commits": []}
|
|
|
|
async def get_commit(self, repo_url: str, token: str, ref: str, client: httpx.AsyncClient) -> dict:
|
|
"""Read one commit's metadata directly, for refs outside the list window."""
|
|
try:
|
|
api_base = self.get_api_base(repo_url)
|
|
headers = self.get_headers(token)
|
|
encoded_path = self._encoded_project(repo_url)
|
|
|
|
response = await client.get(
|
|
f"{api_base}/projects/{encoded_path}/repository/commits/{urllib.parse.quote(ref, safe='')}",
|
|
headers=headers,
|
|
)
|
|
if response.status_code == 404:
|
|
return {"success": False, "message": f"Commit '{ref}' not found in the repository", "commit": None}
|
|
if response.status_code != 200:
|
|
msg = f"Failed to read commit (HTTP {response.status_code}): {self._truncated_response_text(response)}"
|
|
logger.warning("get_commit %s ref=%s: %s", repo_url, ref, msg)
|
|
return {"success": False, "message": msg, "commit": None}
|
|
|
|
try:
|
|
data = response.json()
|
|
except ValueError:
|
|
return {"success": False, "message": "Non-JSON response reading commit", "commit": None}
|
|
sha = data.get("id") if isinstance(data, dict) else None
|
|
if not isinstance(sha, str) or not sha:
|
|
return {"success": False, "message": "Commit response carried no SHA", "commit": None}
|
|
|
|
# GitLab flattens author/date onto the commit, as in list_commits.
|
|
return {
|
|
"success": True,
|
|
"message": "OK",
|
|
"commit": {
|
|
"sha": sha,
|
|
"message": data.get("message") or "",
|
|
"author": data.get("author_name") or "",
|
|
"date": data.get("committed_date") or data.get("created_at") or "",
|
|
},
|
|
}
|
|
|
|
except Exception as e:
|
|
logger.exception("get_commit failed for %s ref=%s", repo_url, ref)
|
|
return {"success": False, "message": f"{type(e).__name__}: {str(e)[:200]}", "commit": None}
|
|
|
|
async def list_tree(
|
|
self,
|
|
repo_url: str,
|
|
token: str,
|
|
ref: str,
|
|
client: httpx.AsyncClient,
|
|
) -> dict:
|
|
"""List blob paths at ``ref`` via /repository/tree, following pagination."""
|
|
try:
|
|
api_base = self.get_api_base(repo_url)
|
|
headers = self.get_headers(token)
|
|
encoded_path = self._encoded_project(repo_url)
|
|
|
|
paths: list[str] = []
|
|
page = 1
|
|
complete = False
|
|
# GitLab's tree endpoint paginates instead of exposing a "truncated"
|
|
# flag, so walk pages until one comes back short. The page cap stops
|
|
# a malformed X-Next-Page loop from spinning forever — and reaching
|
|
# it is a failure, not a result: see the check after the loop.
|
|
while page <= 50:
|
|
response = await client.get(
|
|
f"{api_base}/projects/{encoded_path}/repository/tree",
|
|
headers=headers,
|
|
params={"ref": ref, "recursive": "true", "per_page": 100, "page": page},
|
|
)
|
|
if response.status_code == 404:
|
|
return {
|
|
"success": False,
|
|
"message": f"Commit or tree '{ref}' not found in the repository",
|
|
"paths": [],
|
|
"blob_shas": {},
|
|
}
|
|
if response.status_code != 200:
|
|
msg = (
|
|
f"Failed to list tree (HTTP {response.status_code}): {self._truncated_response_text(response)}"
|
|
)
|
|
logger.warning("list_tree %s ref=%s: %s", repo_url, ref, msg)
|
|
return {"success": False, "message": msg, "paths": [], "blob_shas": {}}
|
|
|
|
try:
|
|
data = response.json()
|
|
except ValueError:
|
|
return {"success": False, "message": "Non-JSON response listing tree", "paths": [], "blob_shas": {}}
|
|
if not isinstance(data, list):
|
|
return {"success": False, "message": "Unexpected shape listing tree", "paths": [], "blob_shas": {}}
|
|
|
|
for item in data:
|
|
if isinstance(item, dict) and item.get("type") == "blob":
|
|
path = item.get("path")
|
|
if isinstance(path, str) and path:
|
|
paths.append(path)
|
|
|
|
if len(data) < 100:
|
|
complete = True
|
|
break
|
|
page += 1
|
|
|
|
if not complete:
|
|
# Falling out of the loop means the last page was full and there
|
|
# are more. Returning success here would hand the restore a
|
|
# silently partial path list, and it would then report the
|
|
# categories it could not see as "not present in this commit" —
|
|
# the same failure GitHub's truncated=true check refuses to allow.
|
|
msg = (
|
|
"Repository tree exceeds the listing limit (more than 5000 files), so the backup "
|
|
"contents cannot be enumerated reliably. Rotate the backup repository."
|
|
)
|
|
logger.warning("list_tree %s ref=%s: %s", repo_url, ref, msg)
|
|
return {"success": False, "message": msg, "paths": [], "blob_shas": {}}
|
|
|
|
# GitLab reads files by path, so there is no blob-SHA map to share.
|
|
return {"success": True, "message": "OK", "paths": sorted(paths), "blob_shas": {}}
|
|
|
|
except Exception as e:
|
|
logger.exception("list_tree failed for %s ref=%s", repo_url, ref)
|
|
return {"success": False, "message": f"{type(e).__name__}: {str(e)[:200]}", "paths": [], "blob_shas": {}}
|
|
|
|
async def fetch_files(
|
|
self,
|
|
repo_url: str,
|
|
token: str,
|
|
ref: str,
|
|
paths: list[str],
|
|
client: httpx.AsyncClient,
|
|
blob_shas: dict[str, str] | None = None,
|
|
) -> dict:
|
|
"""Read ``paths`` at ``ref`` via /repository/files/{path}.
|
|
|
|
``blob_shas`` is accepted for interface parity and ignored: this backend
|
|
addresses files by path, so it never needed the tree listing that makes
|
|
the map worth passing.
|
|
"""
|
|
try:
|
|
api_base = self.get_api_base(repo_url)
|
|
headers = self.get_headers(token)
|
|
encoded_path = self._encoded_project(repo_url)
|
|
|
|
files: dict[str, str] = {}
|
|
for path in paths:
|
|
encoded_file = urllib.parse.quote(path, safe="")
|
|
response = await client.get(
|
|
f"{api_base}/projects/{encoded_path}/repository/files/{encoded_file}",
|
|
headers=headers,
|
|
params={"ref": ref},
|
|
)
|
|
# A path absent from this commit is expected — which categories a
|
|
# backup contains varies by config — so skip rather than fail.
|
|
if response.status_code == 404:
|
|
continue
|
|
if response.status_code != 200:
|
|
msg = (
|
|
f"Failed to read {path} (HTTP {response.status_code}): "
|
|
f"{self._truncated_response_text(response)}"
|
|
)
|
|
logger.warning("fetch_files %s: %s", repo_url, msg)
|
|
return {"success": False, "message": msg, "files": {}}
|
|
|
|
try:
|
|
data = response.json()
|
|
except ValueError:
|
|
return {"success": False, "message": f"Non-JSON response reading {path}", "files": {}}
|
|
if not isinstance(data, dict):
|
|
return {"success": False, "message": f"Unexpected shape reading {path}", "files": {}}
|
|
|
|
content = data.get("content")
|
|
if not isinstance(content, str):
|
|
return {"success": False, "message": f"Missing content reading {path}", "files": {}}
|
|
encoding = data.get("encoding", "base64")
|
|
try:
|
|
if encoding == "base64":
|
|
files[path] = base64.b64decode(content).decode("utf-8")
|
|
elif encoding in ("text", "utf-8", "plain"):
|
|
files[path] = content
|
|
else:
|
|
return {
|
|
"success": False,
|
|
"message": f"Unsupported encoding {encoding!r} reading {path}",
|
|
"files": {},
|
|
}
|
|
except (ValueError, UnicodeDecodeError) as e:
|
|
return {"success": False, "message": f"Could not decode {path}: {type(e).__name__}", "files": {}}
|
|
|
|
return {"success": True, "message": "OK", "files": files}
|
|
|
|
except Exception as e:
|
|
logger.exception("fetch_files failed for %s ref=%s", repo_url, ref)
|
|
return {"success": False, "message": f"{type(e).__name__}: {str(e)[:200]}", "files": {}}
|
|
|
|
async def push_files(
|
|
self,
|
|
repo_url: str,
|
|
token: str,
|
|
branch: str,
|
|
files: dict,
|
|
client: httpx.AsyncClient,
|
|
) -> dict:
|
|
try:
|
|
owner, repo = self.parse_repo_url(repo_url)
|
|
api_base = self.get_api_base(repo_url)
|
|
headers = self.get_headers(token)
|
|
encoded_path = urllib.parse.quote(f"{owner}/{repo}", safe="")
|
|
|
|
encoded_branch = urllib.parse.quote(branch, safe="")
|
|
branch_response = await client.get(
|
|
f"{api_base}/projects/{encoded_path}/repository/branches/{encoded_branch}",
|
|
headers=headers,
|
|
)
|
|
|
|
if branch_response.status_code == 404:
|
|
proj_response = await client.get(f"{api_base}/projects/{encoded_path}", headers=headers)
|
|
if proj_response.status_code != 200:
|
|
return {"status": "failed", "message": "Failed to get project info"}
|
|
|
|
default_branch = proj_response.json().get("default_branch", "main")
|
|
default_encoded = urllib.parse.quote(default_branch, safe="")
|
|
default_response = await client.get(
|
|
f"{api_base}/projects/{encoded_path}/repository/branches/{default_encoded}",
|
|
headers=headers,
|
|
)
|
|
|
|
if default_response.status_code != 200:
|
|
return await self._create_initial_commit(client, headers, api_base, encoded_path, branch, files)
|
|
|
|
create_response = await client.post(
|
|
f"{api_base}/projects/{encoded_path}/repository/branches",
|
|
headers=headers,
|
|
json={"branch": branch, "ref": default_branch},
|
|
)
|
|
if create_response.status_code not in (200, 201):
|
|
return {"status": "failed", "message": f"Failed to create branch: {create_response.status_code}"}
|
|
elif branch_response.status_code != 200:
|
|
return {"status": "failed", "message": f"Failed to check branch: {branch_response.status_code}"}
|
|
|
|
existing_blobs: dict[str, str] = {}
|
|
page = 1
|
|
while True:
|
|
tree_response = await client.get(
|
|
f"{api_base}/projects/{encoded_path}/repository/tree",
|
|
headers=headers,
|
|
params={"recursive": "true", "ref": branch, "per_page": 100, "page": page},
|
|
)
|
|
if tree_response.status_code != 200:
|
|
break
|
|
items = tree_response.json()
|
|
if not items:
|
|
break
|
|
for item in items:
|
|
if item.get("type") == "blob":
|
|
existing_blobs[item["path"]] = item["id"]
|
|
page += 1
|
|
|
|
actions = []
|
|
for path, content in files.items():
|
|
content_str = json.dumps(content, indent=2, default=str)
|
|
content_bytes = content_str.encode("utf-8")
|
|
content_sha = self._blob_sha(content_bytes)
|
|
|
|
if path in existing_blobs and existing_blobs[path] == content_sha:
|
|
continue
|
|
|
|
actions.append(
|
|
{
|
|
"action": "update" if path in existing_blobs else "create",
|
|
"file_path": path,
|
|
"content": base64.b64encode(content_bytes).decode(),
|
|
"encoding": "base64",
|
|
}
|
|
)
|
|
|
|
if not actions:
|
|
return {"status": "skipped", "message": "No changes to commit", "commit_sha": None, "files_changed": 0}
|
|
|
|
commit_message = f"Bambuddy backup - {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S UTC')}"
|
|
commit_response = await client.post(
|
|
f"{api_base}/projects/{encoded_path}/repository/commits",
|
|
headers=headers,
|
|
json={"branch": branch, "commit_message": commit_message, "actions": actions},
|
|
)
|
|
if commit_response.status_code not in (200, 201):
|
|
return {
|
|
"status": "failed",
|
|
"message": f"Failed to create commit: {self._truncated_response_text(commit_response)}",
|
|
}
|
|
|
|
return {
|
|
"status": "success",
|
|
"message": f"Backup successful - {len(actions)} files updated",
|
|
"commit_sha": commit_response.json().get("id"),
|
|
"files_changed": len(actions),
|
|
}
|
|
except Exception as e:
|
|
logger.error("Push to GitLab failed: %s", e)
|
|
return {"status": "failed", "message": str(e), "error": str(e)}
|
|
|
|
async def _create_initial_commit(
|
|
self,
|
|
client: httpx.AsyncClient,
|
|
headers: dict,
|
|
api_base: str,
|
|
encoded_path: str,
|
|
branch: str,
|
|
files: dict,
|
|
) -> dict:
|
|
"""Create the first commit in an empty repository."""
|
|
try:
|
|
actions = []
|
|
for path, content in files.items():
|
|
content_str = json.dumps(content, indent=2, default=str)
|
|
actions.append(
|
|
{
|
|
"action": "create",
|
|
"file_path": path,
|
|
"content": base64.b64encode(content_str.encode()).decode(),
|
|
"encoding": "base64",
|
|
}
|
|
)
|
|
|
|
commit_message = f"Initial Bambuddy backup - {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S UTC')}"
|
|
commit_response = await client.post(
|
|
f"{api_base}/projects/{encoded_path}/repository/commits",
|
|
headers=headers,
|
|
json={"branch": branch, "commit_message": commit_message, "actions": actions, "start_branch": branch},
|
|
)
|
|
if commit_response.status_code not in (200, 201):
|
|
return {
|
|
"status": "failed",
|
|
"message": f"Failed to create initial commit: {self._truncated_response_text(commit_response)}",
|
|
}
|
|
|
|
return {
|
|
"status": "success",
|
|
"message": f"Initial backup created - {len(files)} files",
|
|
"commit_sha": commit_response.json().get("id"),
|
|
"files_changed": len(files),
|
|
}
|
|
except Exception as e:
|
|
return {"status": "failed", "message": str(e)}
|