WIP: add jetbrains updating script

.github/workflows/update-jetbrains.yml

@@ -0,0 +1,24 @@
+name: "Update Jetbrains plugins"
+on:
+  schedule:
+    - cron: '00 2 * * *'
+
+jobs:
+  tests:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2.3.3
+    - name: Install nix
+      uses: cachix/install-nix-action@v11
+      with:
+        nix_path: "${{ matrix.nixPath }}"
+    - name: Show nixpkgs version
+      run: nix-instantiate --eval -E '(import <nixpkgs> {}).lib.version'
+    - name: Generate package list
+      # TODO switch to default nixpkgs channel once nix-build-uncached 1.0.0 is in stable
+      run: nix run -I 'nixpkgs=channel:nixos-unstable' nixpkgs.nix-build-uncached -c nix-build-uncached ci.nix -A cacheOutputs
+    - name: Trigger NUR update
+      if: ${{ matrix.nurRepo != '<YOUR_REPO_NAME>' }}
+      run: curl -XPOST "https://nur-update.herokuapp.com/update?repo=${{ matrix.nurRepo }}"
+      

bin/update-jetbrains-plugins.py

@@ -2,7 +2,6 @@
 #! nix-shell -i python3 -p python3Packages.lxml python3Packages.requests
 
 import argparse
-import os
 import re
 import requests
 import subprocess as sp
@@ -59,20 +58,15 @@ class Build:
         return self.code + "-" + self.version
 
 
-PACKAGE_RE = re.compile("[^0-9A-Za-z._-]")
-HTML_RE = re.compile("<[^>]+/?>")
-
-
 class Plugin:
     def __init__(self, data, category=None):
         self.category = category
         self.name = data.find("name").text
         self.id = data.find("id").text
-        self._description = data.find("description").text
+        self.description = data.find("description").text
         self.url = data.get("url") or data.find("vendor").get("url")
         self.version = data.find("version").text
         self.slug = to_slug(self.name)
-        self.orig_slug = self.slug
 
         self.depends = []
         for depend in data.findall("depends"):
@@ -81,44 +75,19 @@ class Plugin:
     def __repr__(self):
         return f"<Plugin '{self.name}' {self.version}>"
 
-    def description(self):
+    def download_url(self, build, follow_redirect=False):
-        return re.sub(HTML_RE, "", self._description or "").strip()
-
-    def download_url(self, build, deref=True):
-        """
-        Provides the ZIP download URL for this plugin.
-
-        The trivial URL fetches the latest version through a redirect for some
-        build code and provides no locking to a version for the URL. To fetch
-        a stable URL that can be used as a package source, deref must be set
-        (which it is by default). However, this comes at the cost of requiring
-        an HTTP request.
-        """
         id = urllib.parse.quote(self.id)
-        url = f"https://plugins.jetbrains.com/pluginManager?action=download&id={id}&build={build}"
+        return f"https://plugins.jetbrains.com/pluginManager?action=download&id={id}&build={build}"
-        if deref:
-            res = requests.get(url, allow_redirects=not deref)
-            url = "https://plugins.jetbrains.com" + re.sub(
-                "\?.*$", "", res.headers["location"]
-            )
-            if url.endswith("external"):
-                res = requests.get(url, allow_redirects=not deref)
-                url = res.headers["location"]
-        return url
 
     def packagename(self):
-        slug = re.sub(PACKAGE_RE, "", self.slug.lower()).replace(".", "-")
+        slug = self.slug.lower().replace(".", "-")
         if slug[0] in "1234567890":
             return "_" + slug
         else:
             return slug
 
     def filename(self):
-        """
+        return f"{self.slug}-{self.version}.jar"
-        Returns this plugin's filename without an extension. Rely on the
-        download URL to know the extension.
-        """
-        return f"{self.slug}-{self.version}"
 
 
 def list_plugins(build):
@@ -141,23 +110,10 @@ def parse_repository(content):
     return plugins
 
 
-def deduplicate(plugins):
+def prefetch(plugin, build):
-    """
-    Ensures that the plugin list has unique slugs. Modifies the list in-place.
-    """
-    prev = plugins[0]
-    for plugin in plugins[1:]:
-        if plugin.orig_slug == prev.orig_slug:
-            prev.slug = prev.orig_slug + "-" + prev.version.replace(".", "_")
-            plugin.slug = plugin.orig_slug + "-" + plugin.version.replace(".", "_")
-        prev = plugin
-
-
-def prefetch(plugin, build, url=None):
-    if not url:
-        url = plugin.download_url(build)
     res = sp.run(
-        ["nix-prefetch-url", "--name", plugin.filename(), url], capture_output=True,
+        ["nix-prefetch-url", "--name", plugin.filename(), plugin.download_url(build)],
+        capture_output=True,
     )
     if not res.stdout:
         raise IOError(f"nix-prefetch-url {plugin} failed: {res.stderr.decode('utf-8')}")
@@ -169,49 +125,33 @@ def write_packages(outfile, plugins, build):
     outfile.write("{callPackage}:\n{")
 
     for plugin in plugins:
-        src_url = plugin.download_url(build, deref=True)
-        src_ext = os.path.splitext(src_url)[-1]
-
         try:
-            sha = prefetch(plugin, build, src_url)
+            sha = prefetch(plugin, build)
         except IOError as e:
             print(e, file=sys.stderr)
             continue
-
-        build_inputs = []
-        if src_ext == ".zip":
-            build_inputs.append("unzip")
-
         # TODO: Dependencies are provided as package IDs but refer to both
         # internal and external plugins; need to find some way to resolve them
         requires = []
         # TODO: Licenses are actually on the website, but aren't provided in the API
-        license = "lib.licenses.free"
+        license = "lib.license.free"
-
-        call_args = [str(builder), "fetchurl", "lib"]
-        for binput in build_inputs:
-            call_args.append(binput)
 
         outfile.write(
             f"""
-  {plugin.packagename()} = callPackage ({{ {", ".join(sorted(call_args))} }}: {builder} {{
+  {plugin.packagename()} = callPackage ({{ {builder}, fetchurl, lib }}): {builder} {{
     pname = "{plugin.slug}";
     plugname = "{plugin.name}";
     plugid = "{plugin.id}";
     version = "{plugin.version}";
     src = fetchurl {{
-      url = "{src_url}";
+      url = "{plugin.download_url(build)}";
       sha256 = "{sha}";
-      name = "{plugin.filename()}{src_ext}";
+      name = "{plugin.filename()}";
     }};
-    buildInputs = [ {" ".join(build_inputs)} ];
+    packageRequires = {requires};
-    packageRequires = [ {" ".join(requires)} ];
     meta = {{
-      homepage = "{plugin.url or ""}";
+      {f'homepage = {plugin.url};' if plugin.url else ''}
       license = {license};
-      description = ''
-        {plugin.description()}
-      '';
     }};
   }}) {{}};
 """
@@ -239,12 +179,9 @@ def main():
     build = Build(args.package)
     plugins = list_plugins(build)
     plugins.sort(key=lambda p: p.slug)
-    deduplicate(plugins)
-
     if args.number:
         plugins = plugins[: args.number]
 
-    print(f"Generating packages for {len(plugins)} plugins", file=sys.stderr)
     if not args.out:
         write_packages(sys.stdout, plugins, build)
     else:

pkgs/applications/editors/jetbrains/common-plugins.nix

@@ -4,7 +4,7 @@ self:
 
 let
 
-  commonBuild = import ../../../build-support/jetbrains/plugin-old.nix {
+  commonBuild = import ../../../build-support/jetbrains/plugin.nix {
     inherit lib stdenv fetchzip;
     jetbrainsPlatforms = [
       "clion"

pkgs/applications/editors/jetbrains/idea-plugins.nix

@@ -1,19 +1,19 @@
-{ lib, stdenv, variant }:
+{ lib, stdenv, fetchzip }:
 
 self:
 
 let
 
   ideaBuild = import ../../../build-support/jetbrains/plugin.nix {
-    inherit lib stdenv variant;
+    inherit lib stdenv fetchzip;
     jetbrainsPlatforms = [ "idea-community" "idea-ultimate" ];
   };
 
   generateIdea = lib.makeOverridable ({
-    generated ? ./idea-generated.nix
+    idea ? ./manual-idea-packages.nix
   }: let
 
-    imported = import generated {
+    imported = import idea {
       inherit (self) callPackage;
     };
 
@@ -26,3 +26,4 @@ let
   in ideaPlugins // { inherit ideaBuild; });
 
 in generateIdea { }
+

pkgs/applications/editors/jetbrains/wrapper.nix

@@ -21,13 +21,11 @@ in
 assert assertMsg (length badPlugins == 0) errorMsg;
 
 appendToName "with-plugins" (package.overrideAttrs (oldAttrs: {
-  inherit plugins;
+  passthru = { inherit plugins; };
-  # TODO: Remove version from directory name
+  # TODO: Purely aesthetics, but link the plugin to its name instead of hash-name-version
   installPhase = oldAttrs.installPhase + ''
     for plugin in $plugins; do
-      local dirname=$(basename "$plugin")
+      ln -s "$plugin" "$out/$name/plugins/$(basename $plugin)"
-      dirname=''${dirname:33}
-      ln -s "$plugin" "$out/$name/plugins/$dirname"
     done
   '';
 }))

pkgs/build-support/jetbrains/plugin-old.nix

@@ -1,33 +0,0 @@
-{ lib
-, stdenv
-, fetchzip
-, jetbrainsPlatforms
-}:
-
-{ pluginId
-, pname
-, version
-, versionId
-, sha256
-, filename ? "${pname}-${version}.zip"
-}:
-
-stdenv.mkDerivation {
-  inherit pname version;
-
-  src = fetchzip {
-    inherit sha256;
-    url = "https://plugins.jetbrains.com/files/${toString pluginId}/${toString versionId}/${filename}";
-  };
-
-  passthru = { inherit jetbrainsPlatforms; };
-
-  installPhase = ''
-    mkdir $out
-    cp -r * $out/
-  '';
-
-  meta = {
-    homepage = "https://plugins.jetbrains.com/plugin/${pluginId}-${lib.toLower pname}";
-  };
-}

pkgs/build-support/jetbrains/plugin.nix

@@ -1,71 +1,33 @@
 { lib
 , stdenv
-, variant
+, fetchzip
 , jetbrainsPlatforms
 }:
 
-with lib;
+{ pluginId
-
+, pname
-{ pname
 , version
+, versionId
+, sha256
+, filename ? "${pname}-${version}.zip"
+}:
 
-, plugname
+stdenv.mkDerivation {
-, plugid
-
-, buildInputs ? []
-, packageRequires ? []
-, meta ? {}
-
-, ...
-}@args:
-
-let
-
-  defaultMeta = {
-    broken = false;
-    platforms = variant.meta.platforms;
-  } // optionalAttrs ((args.src.meta.homepage or "") != "") {
-    homepage = args.src.meta.homepage;
-  } // optionalAttrs ((args.src.meta.description or "") != "") {
-    description = args.src.meta.description;
-  };
-
-in
-
-stdenv.mkDerivation ({
   inherit pname version;
 
-  unpackCmd = ''
+  src = fetchzip {
-    case "$curSrc" in
+    inherit sha256;
-      *.jar)
+    url = "https://plugins.jetbrains.com/files/${toString pluginId}/${toString versionId}/${filename}";
-        # don't unpack; keep original source filename without the hash
+  };
-        local filename=$(basename "$curSrc")
+
-        filename="''${filename:33}"
+  passthru = { inherit jetbrainsPlatforms; };
-        cp $curSrc $filename
-        chmod +w $filename
-        sourceRoot="."
-        ;;
-      *)
-        _defaultUnpack "$curSrc"
-        ;;
-    esac
-  '';
 
-  # FIXME: Entirely possible this isn't correct for niche plugins;
-  # at the very least there are some plugins that come with JS
   installPhase = ''
-    mkdir -p "$out/lib"
+    mkdir $out
-    find -iname '*.jar' -exec cp {} "$out/lib/" \;
+    cp -r * $out/
   '';
 
-  buildInputs = [ ] ++ packageRequires ++ buildInputs;
+  meta = {
-  propagatedBuildInputs = packageRequires;
+    homepage = "https://plugins.jetbrains.com/plugin/${pluginId}-${lib.toLower pname}";
-
+  };
-  passthru = { inherit jetbrainsPlatforms plugid plugname; };
-
-  doCheck = false;
-
-  meta = defaultMeta // meta;
 }
-
-// removeAttrs args [ "buildInputs" "packageRequires" "meta" ])

pkgs/top-level/jetbrains-plugins.nix

@@ -13,7 +13,7 @@ let
   };
 
   mkIdeaPlugins = import ../applications/editors/jetbrains/idea-plugins.nix {
-    inherit lib stdenv variant;
+    inherit lib stdenv fetchzip;
   };
 
   jetbrainsWithPlugins = import ../applications/editors/jetbrains/wrapper.nix {