Support GitHub /tree/<branch>/<subdir> URLs in project import — fixes the appstore "Remix on CloudPebble" button

cloudpebble/ide/api/project.py

@@ -20,6 +20,7 @@
 from ide.tasks.build import run_compile
 from ide.tasks.gist import import_gist
 from ide.tasks.git import do_import_github
+from ide.utils.github_urls import parse_github_source
 from ide.utils.alloy_templates import list_alloy_templates, build_template_archive
 from ide.utils.c_templates import list_c_templates, build_c_template_archive
 from utils.td_helper import send_td_event
@@ -1029,14 +1030,34 @@ def import_zip(request):
 def import_github(request):
     name = request.POST['name']
     repo = request.POST['repo']
-    branch = request.POST['branch']
+    branch = request.POST.get('branch', '')
     add_remote = (request.POST['add_remote'] == 'true')
-    match = re.match(r'^(?:https?://|git@|git://)?(?:www\.)?github\.com[/:]([\w.-]+)/([\w.-]+?)(?:\.git|/|$)', repo)
-    if match is None:
+    source = parse_github_source(repo)
+    if source is None:
         raise BadRequest(_("Invalid Github URL."))
 
-    github_user = match.group(1)
-    github_project = match.group(2)
+    github_user = source.user
+    github_project = source.project
+    refpath, kind = source.refpath, source.kind
+    if refpath and kind == 'tree' and '/' not in refpath:
+        # A slash-free /tree/<x> is unambiguously a branch or tag — exactly
+        # equivalent to typing it into the branch box, so every existing flow
+        # (add_remote included) stays available.
+        branch, refpath, kind = refpath, None, None
+    elif refpath:
+        # The URL is authoritative for both the ref and the subdirectory (the
+        # celery task resolves the ambiguous split against the repository's
+        # real refs); a separately-typed branch would conflict, so it is
+        # ignored.
+        branch = ''
+        if add_remote:
+            # Deliberately NOT suggesting to link later from settings: the
+            # project model has no notion of a subdirectory, so a linked push
+            # would re-find a project root in the full repository tree and
+            # could overwrite a different project there.
+            if kind == 'commit':
+                raise BadRequest(_("Linking a repository is not supported for commit imports."))
+            raise BadRequest(_("Linking a repository is not supported for subdirectory imports."))
 
     try:
         project = Project.objects.create(owner=request.user, name=name)
@@ -1045,10 +1066,11 @@ def import_github(request):
 
     if add_remote:
         project.github_repo = "%s/%s" % (github_user, github_project)
-        project.github_branch = branch
+        project.github_branch = branch or None
         project.save()
 
-    task = do_import_github.delay(project.id, github_user, github_project, branch, delete_project=True)
+    task = do_import_github.delay(project.id, github_user, github_project, branch, delete_project=True,
+                                  github_refpath=refpath, github_kind=kind)
     return {'task_id': task.task_id, 'project_id': project.id}
 
 

cloudpebble/ide/static/ide/js/__tests__/github-import-prefill.test.js

@@ -0,0 +1,106 @@
+import { describe, it, expect, vi } from 'vitest';
+import { readFileSync } from 'fs';
+import { resolve } from 'path';
+
+// project_list.js is one $(function(){...}) block; the jQuery mock runs the
+// callback immediately, so loading the file executes the deep-link prefill.
+// Every jQuery method is a chainable no-op except val(), which records per
+// selector — enough to assert what lands in the import dialog's fields.
+function makeJqueryMock() {
+    var elements = {};
+    var $ = function(selector) {
+        if (typeof selector === 'function') {
+            selector();
+            return $;
+        }
+        if (!elements[selector]) {
+            var store = { value: undefined };
+            var el = new Proxy(store, {
+                get: function(target, prop) {
+                    if (prop === 'val') {
+                        return function(v) {
+                            if (v === undefined) return target.value;
+                            target.value = v;
+                            return el;
+                        };
+                    }
+                    if (prop === 'text') return function() { return ''; };
+                    if (prop === 'length') return 0;
+                    if (prop === '_isMock') return true;
+                    return function() { return el; };
+                }
+            });
+            elements[selector] = el;
+        }
+        return elements[selector];
+    };
+    $.Deferred = vi.fn();
+    $.elements = elements;
+    return $;
+}
+
+function loadWithPath(pathname) {
+    var code = readFileSync(resolve(__dirname, '..', 'project_list.js'), 'utf8');
+    var $ = makeJqueryMock();
+    var fn = new Function(
+        '$', 'jQuery', 'gettext', 'jquery_csrf_setup', 'ga', 'Ajax', 'CloudPebble', 'location',
+        code
+    );
+    fn($, $, function(s) { return s; }, vi.fn(), vi.fn(), {}, {}, { pathname: pathname });
+    return {
+        name: $('#import-github-name').val(),
+        url: $('#import-github-url').val(),
+        branch: $('#import-github-branch').val()
+    };
+}
+
+describe('GitHub import deep-link prefill', () => {
+    it('hands a /tree/<branch>/<subdir> URL (the appstore Remix shape) to the server whole', () => {
+        var fields = loadWithPath('/ide/import/github/emindeniz99/pebble-signals/tree/main/faces/slothvec');
+        expect(fields.url).toBe('github.com/emindeniz99/pebble-signals/tree/main/faces/slothvec');
+        expect(fields.name).toBe('slothvec');
+        // The URL is authoritative for the ref — the branch box stays empty.
+        expect(fields.branch).toBeUndefined();
+    });
+
+    it('suggests the repo name when a /tree/ URL has no subdirectory', () => {
+        var fields = loadWithPath('/ide/import/github/user/repo/tree/main');
+        expect(fields.url).toBe('github.com/user/repo/tree/main');
+        expect(fields.name).toBe('repo');
+        expect(fields.branch).toBeUndefined();
+    });
+
+    it('suggests the repo name for /blob/ and /commit/ URLs (tail is a file or SHA, not a name)', () => {
+        var blob = loadWithPath('/ide/import/github/user/repo/blob/main/src/app.js');
+        expect(blob.url).toBe('github.com/user/repo/blob/main/src/app.js');
+        expect(blob.name).toBe('repo');
+        var commit = loadWithPath('/ide/import/github/user/repo/commit/abc123');
+        expect(commit.url).toBe('github.com/user/repo/commit/abc123');
+        expect(commit.name).toBe('repo');
+    });
+
+    it('keeps the legacy /<user>/<repo>/<branch> deep-link contract', () => {
+        var fields = loadWithPath('/ide/import/github/user/repo/dev');
+        expect(fields.url).toBe('github.com/user/repo');
+        expect(fields.name).toBe('user');
+        expect(fields.branch).toBe('dev');
+    });
+
+    it('keeps legacy slashed-branch deep links', () => {
+        var fields = loadWithPath('/ide/import/github/user/repo/feat/x');
+        expect(fields.url).toBe('github.com/user/repo');
+        expect(fields.branch).toBe('feat/x');
+    });
+
+    it('leaves the branch empty for a plain repo link (server imports the default branch)', () => {
+        var fields = loadWithPath('/ide/import/github/user/repo');
+        expect(fields.url).toBe('github.com/user/repo');
+        expect(fields.branch).toBeUndefined();
+    });
+
+    it('does nothing outside the import deep link', () => {
+        var fields = loadWithPath('/ide/');
+        expect(fields.url).toBeUndefined();
+        expect(fields.name).toBeUndefined();
+    });
+});

cloudpebble/ide/static/ide/js/project_list.js

@@ -178,14 +178,14 @@ $(function() {
             active_set.find('.errors').removeClass('hide').text(gettext("You must specify a project name."));
             return;
         }
-        // This is identical to the regex used on the server.
+        // A prefix check only — the server (ide.utils.github_urls) is the
+        // authority and also understands /tree/<ref>/<subdir> and /blob/ URLs.
         if(!/^(?:https?:\/\/|git@|git:\/\/)?(?:www\.)?github\.com[\/:]([\w.-]+)\/([\w.-]+?)(?:\.git|\/|$)/.test(url)) {
             active_set.find('.errors').removeClass('hide').text(gettext("You must specify a complete GitHub project URL"));
             return;
         }
-        if(branch.length == 0) {
-            branch = 'master';
-        }
+        // An empty branch imports the repository's default branch (the old
+        // hardcoded 'master' fallback broke every main-default repository).
         disable_import_controls();
         active_set.find('.progress').removeClass('hide');
         do_import(Ajax.Post('/ide/import/github', {
@@ -227,10 +227,25 @@ $(function() {
     if (path.indexOf('/ide/import/github/') === 0) {
         var parts = path.substr(1).split('/');
         $('#import-prompt').modal();
-        $('#import-github-name').val(parts[3]);
-        $('#import-github-url').val('github.com/' + parts[3] + '/' + parts[4]);
-        if (parts.length > 5) {
-            $('#import-github-branch').val(parts.slice(5).join('/'));
+        var tail = parts.slice(5).filter(function(p) { return p.length > 0; });
+        if (tail.length > 1 && (tail[0] == 'tree' || tail[0] == 'blob' || tail[0] == 'commit')) {
+            // A GitHub web URL shape (e.g. the appstore's "Remix" button:
+            // .../import/github/<user>/<repo>/tree/<branch>/<subdir>). Hand
+            // the whole thing to the server, which resolves branch vs
+            // subdirectory against the repository's real refs.
+            // Suggest the subdirectory name for /tree/ URLs (it may still be
+            // part of a slashed branch name — the field is editable and the
+            // server resolves the truth); blob/commit tails are a filename or
+            // a SHA, so the repository name is the sane suggestion there.
+            var suggested = (tail[0] == 'tree' && tail.length > 2) ? tail[tail.length - 1] : parts[4];
+            $('#import-github-name').val(suggested);
+            $('#import-github-url').val('github.com/' + parts.slice(3).join('/'));
+        } else {
+            $('#import-github-name').val(parts[3]);
+            $('#import-github-url').val('github.com/' + parts[3] + '/' + parts[4]);
+            if (tail.length) {
+                $('#import-github-branch').val(tail.join('/'));
+            }
         }
         $('a[href=#import-github]').tab('show');
     }

cloudpebble/ide/tasks/archive.py

@@ -25,6 +25,11 @@
 
 logger = logging.getLogger(__name__)
 
+# When a root_hint narrows an import to one project, the 400-entry limit
+# applies to that subtree — but the whole archive still needs a ceiling to
+# bound the root-finding scan over the repository it lives in.
+HINTED_ARCHIVE_SCAN_LIMIT = 50000
+
 
 def _public_export_url(path):
     return "/ide/export/%s" % path.lstrip('/')
@@ -145,7 +150,7 @@ def ends_with_any(s, options):
 
 
 @shared_task(acks_late=True)
-def do_import_archive(project_id, archive, delete_project=False, wipe_existing=False):
+def do_import_archive(project_id, archive, delete_project=False, wipe_existing=False, root_hint=None):
     project = Project.objects.get(pk=project_id)
     try:
         with tempfile.NamedTemporaryFile(suffix='.zip') as archive_file:
@@ -168,11 +173,18 @@ def do_import_archive(project_id, archive, delete_project=False, wipe_existing=F
                 WORKER_SRC_DIR = 'worker_src/'
                 INCLUDE_SRC_DIR = 'include/'
 
-                if len(contents) > 400:
+                if len(contents) > (400 if root_hint is None else HINTED_ARCHIVE_SCAN_LIMIT):
                     raise InvalidProjectArchiveException("Too many files in zip file.")
 
                 archive_items = [ArchiveProjectItem(z, x) for x in contents]
-                base_dir, manifest_item = find_project_root_and_manifest(archive_items)
+                base_dir, manifest_item = find_project_root_and_manifest(archive_items, root_hint=root_hint)
+                if root_hint is not None:
+                    # A subdirectory import only takes files under the hinted
+                    # project, so the size limit applies to that subtree — not
+                    # to the whole repository the project happens to live in.
+                    contents = [entry for entry in contents if entry.filename.startswith(base_dir)]
+                    if len(contents) > 400:
+                        raise InvalidProjectArchiveException("Too many files in the project directory.")
                 dir_end = len(base_dir)
 
                 def make_valid_filename(zip_entry):

cloudpebble/ide/tasks/git.py

@@ -1,9 +1,11 @@
 import base64
 import io
+import posixpath
 from concurrent.futures import ThreadPoolExecutor
 from io import BytesIO
 from urllib.request import urlopen, Request
 from urllib.error import URLError, HTTPError
+from urllib.parse import quote
 import json
 import os
 import logging
@@ -22,6 +24,7 @@
 from ide.tasks import do_import_archive, run_compile
 from ide.tasks.archive import get_filename_variant
 from ide.utils.git import git_sha, git_blob
+from ide.utils.github_urls import split_ref_and_path, normalize_subpath
 from ide.utils.project import find_project_root_and_manifest, BaseProjectItem, InvalidProjectArchiveException, MANIFEST_KINDS
 from ide.utils.sdk import generate_manifest_dict, generate_manifest, generate_wscript_file, load_manifest_dict, manifest_name_for_project
 from utils.td_helper import send_td_event
@@ -40,7 +43,8 @@ def exception_reason(error):
 
 
 @shared_task(acks_late=True)
-def do_import_github(project_id, github_user, github_project, github_branch, delete_project=False):
+def do_import_github(project_id, github_user, github_project, github_branch, delete_project=False,
+                     github_refpath=None, github_kind=None):
     project = None
     user = None
     try:
@@ -50,7 +54,22 @@ def do_import_github(project_id, github_user, github_project, github_branch, del
         except:
             pass
 
-        url = "https://github.com/%s/%s/archive/%s.zip" % (github_user, github_project, github_branch)
+        github_path = ''
+        if github_refpath:
+            # A /tree/ or /blob/ URL: the branch and the subdirectory are one
+            # ambiguous string until matched against the repository's refs.
+            github_branch, github_path = resolve_ref_and_path(
+                user, github_user, github_project, github_refpath, github_kind)
+        if not github_branch:
+            # codeload resolves HEAD to the repository's default branch, so an
+            # empty branch imports what visitors see — instead of the old
+            # hardcoded 'master', which broke every main-default repository.
+            github_branch = 'HEAD'
+
+        # quote() the ref: slashes are real URL separators here, but a branch
+        # name may legally contain '#', '?' or spaces, which would otherwise
+        # truncate the request path.
+        url = "https://github.com/%s/%s/archive/%s.zip" % (github_user, github_project, quote(github_branch, safe='/'))
         auth_url = get_authenticated_archive_url(user, github_user, github_project, github_branch)
         archive = None
 
@@ -68,7 +87,7 @@ def do_import_github(project_id, github_user, github_project, github_branch, del
                 % (github_user, github_project, github_branch)
             )
 
-        return do_import_archive(project_id, archive.read())
+        return do_import_archive(project_id, archive.read(), root_hint=github_path or None)
     except Exception as e:
         if delete_project and project is not None:
             try:
@@ -80,12 +99,74 @@ def do_import_github(project_id, github_user, github_project, github_branch, del
                 'reason': exception_reason(e),
                 'github_user': github_user,
                 'github_project': github_project,
-                'github_branch': github_branch
+                'github_branch': github_branch,
+                'github_refpath': github_refpath
             }
         }, user=user)
         raise
 
 
+def resolve_ref_and_path(user, github_user, github_project, refpath, kind):
+    """ Resolve a /tree/ or /blob/ remainder ("<ref>[/<path>]") into a real
+    (ref, subdirectory) pair. Branch names may contain slashes, so the split
+    is decided against the repository's actual refs — via the GitHub API when
+    possible, else by probing which archive actually exists (longest candidate
+    first), the way gitpick and gitingest resolve GitHub web URLs. """
+    if kind == 'commit':
+        return refpath, ''
+
+    # A self-qualified refs/heads/... or refs/tags/... spelling pins the
+    # namespace; strip it HERE so the probe fallback below builds candidates
+    # without the qualifier (probing 'refs/heads/refs/heads/x' can never hit).
+    namespaces = ('heads', 'tags')
+    qualified = refpath.split('/')
+    if len(qualified) >= 3 and qualified[0] == 'refs' and qualified[1] in namespaces:
+        namespaces = (qualified[1],)
+        refpath = '/'.join(qualified[2:])
+
+    ref, path = None, None
+    ref_names = get_ref_names(user, github_user, github_project)
+    if ref_names is not None:
+        ref, path = split_ref_and_path(refpath, ref_names)
+    elif '/' in refpath:
+        # No API access (rate limit, private repo without a token): probe the
+        # public archive endpoint. Longest first, so 'feature/foo/src' tries
+        # branch 'feature/foo' before branch 'feature'. The refs-qualified
+        # codeload form MUST be used here: bare archive/<x>.zip answers 200
+        # for junk like 'main/anything', while zip/refs/heads/<x> is strict
+        # (measured: refs/heads/main 200, refs/heads/main/faces 404).
+        parts = refpath.split('/')
+        start = len(parts) if kind == 'tree' else len(parts) - 1
+        for i in range(start, 0, -1):
+            candidate = '/'.join(parts[:i])
+            if any(file_exists("https://codeload.github.com/%s/%s/zip/refs/%s/%s"
+                               % (github_user, github_project, refkind, quote(candidate, safe='/')))
+                   for refkind in namespaces):
+                ref, path = candidate, '/'.join(parts[i:])
+                break
+    if ref is None:
+        ref, path = split_ref_and_path(refpath, None)
+
+    if kind == 'blob':
+        # A blob URL names a file; the project is the directory it lives in.
+        path = posixpath.dirname(path)
+    try:
+        path = normalize_subpath(path)
+    except ValueError:
+        raise Exception("Invalid project path in GitHub URL: '%s'" % refpath)
+    return ref, path
+
+
+def get_ref_names(user, github_user, github_project):
+    """ Branch + tag names, or None when the API is unavailable. """
+    try:
+        g = get_github(user) if user is not None else Github()
+        repo = g.get_repo("%s/%s" % (github_user, github_project))
+        return [b.name for b in repo.get_branches()] + [t.name for t in repo.get_tags()]
+    except Exception:
+        return None
+
+
 def file_exists(url):
     request = Request(url)
     request.get_method = lambda: 'HEAD'