Skip to content

Commit

Permalink
fix none organization name
Browse files Browse the repository at this point in the history
  • Loading branch information
xiaoyao9184 committed Jan 7, 2025
1 parent 9a5a996 commit bba745d
Show file tree
Hide file tree
Showing 8 changed files with 39 additions and 18 deletions.
13 changes: 13 additions & 0 deletions .vscode/tasks.json
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,19 @@
"type": "shell",
"command": "docker build -t xiaoyao9184/olah:0.3.3 -f ./docker/build@pypi/dockerfile .",
},
{
"label": "huggingface-cli: download bert-base-uncased",
"type": "shell",
"options": {
"cwd": "${workspaceFolder}",
"env": {
"HF_ENDPOINT": "http://localhost:18090",
"HF_HUB_ETAG_TIMEOUT": "100",
"HF_HUB_DOWNLOAD_TIMEOUT": "100"
}
},
"command": "huggingface-cli download bert-base-uncased --revision main --cache-dir ./cache/huggingface/hub"
},
{
"label": "huggingface-cli: download cais/mmlu",
"type": "shell",
Expand Down
3 changes: 2 additions & 1 deletion olah/proxy/commits.py
Original file line number Diff line number Diff line change
Expand Up @@ -80,10 +80,11 @@ async def commits_generator(
if authorization is not None:
headers["authorization"] = authorization

org_repo = get_org_repo(org, repo)
# save
repos_path = app.app_settings.config.repos_path
save_dir = os.path.join(
repos_path, f"api/{repo_type}/{org}/{repo}/commits/{commit}"
repos_path, f"api/{repo_type}/{org_repo}/commits/{commit}"
)
save_path = os.path.join(save_dir, f"commits_{method}.json")

Expand Down
8 changes: 4 additions & 4 deletions olah/proxy/files.py
Original file line number Diff line number Diff line change
Expand Up @@ -468,10 +468,10 @@ async def file_get_generator(
# save
repos_path = app.app_settings.config.repos_path
head_path = os.path.join(
repos_path, f"heads/{repo_type}/{org}/{repo}/resolve/{commit}/{file_path}"
repos_path, f"heads/{repo_type}/{org_repo}/resolve/{commit}/{file_path}"
)
save_path = os.path.join(
repos_path, f"files/{repo_type}/{org}/{repo}/resolve/{commit}/{file_path}"
repos_path, f"files/{repo_type}/{org_repo}/resolve/{commit}/{file_path}"
)
make_dirs(head_path)
make_dirs(save_path)
Expand Down Expand Up @@ -522,10 +522,10 @@ async def cdn_file_get_generator(
# save
repos_path = app.app_settings.config.repos_path
head_path = os.path.join(
repos_path, f"heads/{repo_type}/{org}/{repo}/cdn/{file_hash}"
repos_path, f"heads/{repo_type}/{org_repo}/cdn/{file_hash}"
)
save_path = os.path.join(
repos_path, f"files/{repo_type}/{org}/{repo}/cdn/{file_hash}"
repos_path, f"files/{repo_type}/{org_repo}/cdn/{file_hash}"
)
make_dirs(head_path)
make_dirs(save_path)
Expand Down
3 changes: 2 additions & 1 deletion olah/proxy/meta.py
Original file line number Diff line number Diff line change
Expand Up @@ -76,10 +76,11 @@ async def meta_generator(
if authorization is not None:
headers["authorization"] = authorization

org_repo = get_org_repo(org, repo)
# save
repos_path = app.app_settings.config.repos_path
save_dir = os.path.join(
repos_path, f"api/{repo_type}/{org}/{repo}/revision/{commit}"
repos_path, f"api/{repo_type}/{org_repo}/revision/{commit}"
)
save_path = os.path.join(save_dir, f"meta_{method}.json")
make_dirs(save_path)
Expand Down
3 changes: 2 additions & 1 deletion olah/proxy/pathsinfo.py
Original file line number Diff line number Diff line change
Expand Up @@ -72,13 +72,14 @@ async def pathsinfo_generator(
if authorization is not None:
headers["authorization"] = authorization

org_repo = get_org_repo(org, repo)
# save
repos_path = app.app_settings.config.repos_path

final_content = []
for path in paths:
save_dir = os.path.join(
repos_path, f"api/{repo_type}/{org}/{repo}/paths-info/{commit}/{path}"
repos_path, f"api/{repo_type}/{org_repo}/paths-info/{commit}/{path}"
)

save_path = os.path.join(save_dir, f"paths-info_{method}.json")
Expand Down
3 changes: 2 additions & 1 deletion olah/proxy/tree.py
Original file line number Diff line number Diff line change
Expand Up @@ -82,10 +82,11 @@ async def tree_generator(
if authorization is not None:
headers["authorization"] = authorization

org_repo = get_org_repo(org, repo)
# save
repos_path = app.app_settings.config.repos_path
save_dir = os.path.join(
repos_path, f"api/{repo_type}/{org}/{repo}/tree/{commit}/{path}"
repos_path, f"api/{repo_type}/{org_repo}/tree/{commit}/{path}"
)
save_path = os.path.join(save_dir, f"tree_{method}_recursive_{recursive}_expand_{expand}.json")

Expand Down
12 changes: 6 additions & 6 deletions olah/server.py
Original file line number Diff line number Diff line change
Expand Up @@ -193,7 +193,7 @@ async def meta_proxy_common(repo_type: str, org: str, repo: str, commit: str, me
# Check Mirror Path
for mirror_path in app.app_settings.config.mirrors_path:
try:
git_path = os.path.join(mirror_path, repo_type, org, repo)
git_path = os.path.join(mirror_path, repo_type, org or '', repo)
if os.path.exists(git_path):
local_repo = LocalMirrorRepo(git_path, repo_type, org, repo)
meta_data = local_repo.get_meta(commit)
Expand Down Expand Up @@ -370,7 +370,7 @@ async def tree_proxy_common(
# Check Mirror Path
for mirror_path in app.app_settings.config.mirrors_path:
try:
git_path = os.path.join(mirror_path, repo_type, org, repo)
git_path = os.path.join(mirror_path, repo_type, org or '', repo)
if os.path.exists(git_path):
local_repo = LocalMirrorRepo(git_path, repo_type, org, repo)
tree_data = local_repo.get_tree(commit, path, recursive=recursive, expand=expand)
Expand Down Expand Up @@ -514,7 +514,7 @@ async def pathsinfo_proxy_common(repo_type: str, org: str, repo: str, commit: st
# Check Mirror Path
for mirror_path in app.app_settings.config.mirrors_path:
try:
git_path = os.path.join(mirror_path, repo_type, org, repo)
git_path = os.path.join(mirror_path, repo_type, org or '', repo)
if os.path.exists(git_path):
local_repo = LocalMirrorRepo(git_path, repo_type, org, repo)
pathsinfo_data = local_repo.get_pathinfos(commit, paths)
Expand Down Expand Up @@ -641,7 +641,7 @@ async def commits_proxy_common(repo_type: str, org: str, repo: str, commit: str,
# Check Mirror Path
for mirror_path in app.app_settings.config.mirrors_path:
try:
git_path = os.path.join(mirror_path, repo_type, org, repo)
git_path = os.path.join(mirror_path, repo_type, org or '', repo)
if os.path.exists(git_path):
local_repo = LocalMirrorRepo(git_path, repo_type, org, repo)
commits_data = local_repo.get_commits(commit)
Expand Down Expand Up @@ -789,7 +789,7 @@ async def file_head_common(
# Check Mirror Path
for mirror_path in app.app_settings.config.mirrors_path:
try:
git_path = os.path.join(mirror_path, repo_type, org, repo)
git_path = os.path.join(mirror_path, repo_type, org or '', repo)
if os.path.exists(git_path):
local_repo = LocalMirrorRepo(git_path, repo_type, org, repo)
head = local_repo.get_file_head(commit_hash=commit, path=file_path)
Expand Down Expand Up @@ -924,7 +924,7 @@ async def file_get_common(
# Check Mirror Path
for mirror_path in app.app_settings.config.mirrors_path:
try:
git_path = os.path.join(mirror_path, repo_type, org, repo)
git_path = os.path.join(mirror_path, repo_type, org or '', repo)
if os.path.exists(git_path):
local_repo = LocalMirrorRepo(git_path, repo_type, org, repo)
content_stream = local_repo.get_file(commit_hash=commit, path=file_path)
Expand Down
12 changes: 8 additions & 4 deletions olah/utils/repo_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -74,8 +74,9 @@ def get_meta_save_path(
The path to save the meta.json file as a string.
"""
org_repo = get_org_repo(org, repo)
return os.path.join(
repos_path, f"api/{repo_type}/{org}/{repo}/revision/{commit}/meta_get.json"
repos_path, f"api/{repo_type}/{org_repo}/revision/{commit}/meta_get.json"
)


Expand All @@ -95,7 +96,8 @@ def get_meta_save_dir(
The directory path to save the meta.json file as a string.
"""
return os.path.join(repos_path, f"api/{repo_type}/{org}/{repo}/revision")
org_repo = get_org_repo(org, repo)
return os.path.join(repos_path, f"api/{repo_type}/{org_repo}/revision")


def get_file_save_path(
Expand All @@ -121,8 +123,9 @@ def get_file_save_path(
The path to save the file as a string.
"""
org_repo = get_org_repo(org, repo)
return os.path.join(
repos_path, f"heads/{repo_type}/{org}/{repo}/resolve_head/{commit}/{file_path}"
repos_path, f"heads/{repo_type}/{org_repo}/resolve_head/{commit}/{file_path}"
)


Expand Down Expand Up @@ -183,8 +186,9 @@ async def get_newest_commit_hf(
The newest commit hash as a string, or None if it cannot be obtained.
"""
org_repo = get_org_repo(org, repo)
url = urljoin(
app.app_settings.config.hf_url_base(), f"/api/{repo_type}/{org}/{repo}"
app.app_settings.config.hf_url_base(), f"/api/{repo_type}/{org_repo}"
)
if app.app_settings.config.offline:
return await get_newest_commit_hf_offline(app, repo_type, org, repo)
Expand Down

0 comments on commit bba745d

Please sign in to comment.