256 lines
7.3 KiB
Python
256 lines
7.3 KiB
Python
|
#! /usr/bin/env nix-shell
|
||
|
|
#! nix-shell -i python3 -p python3Packages.lxml python3Packages.requests
|
||
|
|
|
||
|
|
import argparse
|
||
|
|
import os
|
||
|
|
import re
|
||
|
|
import requests
|
||
|
|
import subprocess as sp
|
||
|
|
import sys
|
||
|
|
import urllib
|
||
|
|
|
||
|
|
from lxml import etree
|
||
|
|
|
||
|
|
# https://plugins.jetbrains.com/docs/marketplace/product-codes.html
|
||
|
|
PRODUCT_CODE = {
|
||
|
|
"clion": "CL",
|
||
|
|
"datagrip": "DB",
|
||
|
|
"goland": "GO",
|
||
|
|
"idea-community": "IC",
|
||
|
|
"idea-ultimate": "IU",
|
||
|
|
"phpstorm": "PS",
|
||
|
|
"pycharm-community": "PC",
|
||
|
|
"pycharm-professional": "PY",
|
||
|
|
"rider": "RD",
|
||
|
|
"ruby-mine": "RM",
|
||
|
|
"webstorm": "WS",
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
def to_slug(name):
|
||
|
|
slug = name.replace(" ", "-").lstrip(".")
|
||
|
|
for char in ",/;'\\<>:\"|!@#$%^&*()":
|
||
|
|
slug = slug.replace(char, "")
|
||
|
|
return slug
|
||
|
|
|
||
|
|
|
||
|
|
class Build:
|
||
|
|
"""
|
||
|
|
Transforms a Nixpkgs derivation name into a Jetbrains product code. For
|
||
|
|
example:
|
||
|
|
|
||
|
|
idea-community-2019.3.2 -> IC-193.2
|
||
|
|
"""
|
||
|
|
|
||
|
|
def __init__(self, name):
|
||
|
|
m = re.search("([0-9]+\.?)+$", name)
|
||
|
|
version = m.group(0)
|
||
|
|
code = PRODUCT_CODE[name.replace("-" + version, "")]
|
||
|
|
vparts = version.split(".")
|
||
|
|
version = vparts[0][-2:] + vparts[1] + "." + vparts[2]
|
||
|
|
self.code = code
|
||
|
|
self.version = version
|
||
|
|
self.package = name.split("-")[0]
|
||
|
|
|
||
|
|
def builder(self):
|
||
|
|
return self.package + "Build"
|
||
|
|
|
||
|
|
def __repr__(self):
|
||
|
|
return self.code + "-" + self.version
|
||
|
|
|
||
|
|
|
||
|
|
PACKAGE_RE = re.compile("[^0-9A-Za-z._-]")
|
||
|
|
HTML_RE = re.compile("<[^>]+/?>")
|
||
|
|
|
||
|
|
|
||
|
|
class Plugin:
|
||
|
|
def __init__(self, data, category=None):
|
||
|
|
self.category = category
|
||
|
|
self.name = data.find("name").text
|
||
|
|
self.id = data.find("id").text
|
||
|
|
self._description = data.find("description").text
|
||
|
|
self.url = data.get("url") or data.find("vendor").get("url")
|
||
|
|
self.version = data.find("version").text
|
||
|
|
self.slug = to_slug(self.name)
|
||
|
|
self.orig_slug = self.slug
|
||
|
|
|
||
|
|
self.depends = []
|
||
|
|
for depend in data.findall("depends"):
|
||
|
|
self.depends.append(depend.text)
|
||
|
|
|
||
|
|
def __repr__(self):
|
||
|
|
return f"<Plugin '{self.name}' {self.version}>"
|
||
|
|
|
||
|
|
def description(self):
|
||
|
|
return re.sub(HTML_RE, "", self._description or "").strip()
|
||
|
|
|
||
|
|
def download_url(self, build, deref=True):
|
||
|
|
"""
|
||
|
|
Provides the ZIP download URL for this plugin.
|
||
|
|
|
||
|
|
The trivial URL fetches the latest version through a redirect for some
|
||
|
|
build code and provides no locking to a version for the URL. To fetch
|
||
|
|
a stable URL that can be used as a package source, deref must be set
|
||
|
|
(which it is by default). However, this comes at the cost of requiring
|
||
|
|
an HTTP request.
|
||
|
|
"""
|
||
|
|
id = urllib.parse.quote(self.id)
|
||
|
|
url = f"https://plugins.jetbrains.com/pluginManager?action=download&id={id}&build={build}"
|
||
|
|
if deref:
|
||
|
|
res = requests.get(url, allow_redirects=not deref)
|
||
|
|
url = "https://plugins.jetbrains.com" + re.sub(
|
||
|
|
"\?.*$", "", res.headers["location"]
|
||
|
|
)
|
||
|
|
if url.endswith("external"):
|
||
|
|
res = requests.get(url, allow_redirects=not deref)
|
||
|
|
url = res.headers["location"]
|
||
|
|
return url
|
||
|
|
|
||
|
|
def packagename(self):
|
||
|
|
slug = re.sub(PACKAGE_RE, "", self.slug.lower()).replace(".", "-")
|
||
|
|
if slug[0] in "1234567890":
|
||
|
|
return "_" + slug
|
||
|
|
else:
|
||
|
|
return slug
|
||
|
|
|
||
|
|
def filename(self):
|
||
|
|
"""
|
||
|
|
Returns this plugin's filename without an extension. Rely on the
|
||
|
|
download URL to know the extension.
|
||
|
|
"""
|
||
|
|
return f"{self.slug}-{self.version}"
|
||
|
|
|
||
|
|
|
||
|
|
def list_plugins(build):
|
||
|
|
"""
|
||
|
|
Lists all plugins for the specified build code.
|
||
|
|
|
||
|
|
https://plugins.jetbrains.com/docs/marketplace/plugins-list.html
|
||
|
|
"""
|
||
|
|
resp = requests.get(f"https://plugins.jetbrains.com/plugins/list/?build={build}")
|
||
|
|
return parse_repository(resp.content)
|
||
|
|
|
||
|
|
|
||
|
|
def parse_repository(content):
|
||
|
|
tree = etree.XML(content)
|
||
|
|
plugins = []
|
||
|
|
for cat in tree.findall("category"):
|
||
|
|
cat_name = cat.get("name")
|
||
|
|
for plugin in cat.findall("idea-plugin"):
|
||
|
|
plugins.append(Plugin(plugin, cat_name))
|
||
|
|
return plugins
|
||
|
|
|
||
|
|
|
||
|
|
def deduplicate(plugins):
|
||
|
|
"""
|
||
|
|
Ensures that the plugin list has unique slugs. Modifies the list in-place.
|
||
|
|
"""
|
||
|
|
prev = plugins[0]
|
||
|
|
for plugin in plugins[1:]:
|
||
|
|
if plugin.orig_slug == prev.orig_slug:
|
||
|
|
prev.slug = prev.orig_slug + "-" + prev.version.replace(".", "_")
|
||
|
|
plugin.slug = plugin.orig_slug + "-" + plugin.version.replace(".", "_")
|
||
|
|
prev = plugin
|
||
|
|
|
||
|
|
|
||
|
|
def prefetch(plugin, build, url=None):
|
||
|
|
if not url:
|
||
|
|
url = plugin.download_url(build)
|
||
|
|
res = sp.run(
|
||
|
|
["nix-prefetch-url", "--name", plugin.filename(), url], capture_output=True,
|
||
|
|
)
|
||
|
|
if not res.stdout:
|
||
|
|
raise IOError(f"nix-prefetch-url {plugin} failed: {res.stderr.decode('utf-8')}")
|
||
|
|
return res.stdout.decode("utf-8").strip()
|
||
|
|
|
||
|
|
|
||
|
|
def write_packages(outfile, plugins, build):
|
||
|
|
builder = build.builder()
|
||
|
|
outfile.write("{callPackage}:\n{")
|
||
|
|
|
||
|
|
for plugin in plugins:
|
||
|
|
src_url = plugin.download_url(build, deref=True)
|
||
|
|
src_ext = os.path.splitext(src_url)[-1]
|
||
|
|
|
||
|
|
try:
|
||
|
|
sha = prefetch(plugin, build, src_url)
|
||
|
|
except IOError as e:
|
||
|
|
print(e, file=sys.stderr)
|
||
|
|
continue
|
||
|
|
|
||
|
|
build_inputs = []
|
||
|
|
if src_ext == ".zip":
|
||
|
|
build_inputs.append("unzip")
|
||
|
|
|
||
|
|
# TODO: Dependencies are provided as package IDs but refer to both
|
||
|
|
# internal and external plugins; need to find some way to resolve them
|
||
|
|
requires = []
|
||
|
|
# TODO: Licenses are actually on the website, but aren't provided in the API
|
||
|
|
license = "lib.licenses.free"
|
||
|
|
|
||
|
|
call_args = [str(builder), "fetchurl", "lib"]
|
||
|
|
for binput in build_inputs:
|
||
|
|
call_args.append(binput)
|
||
|
|
|
||
|
|
outfile.write(
|
||
|
|
f"""
|
||
|
|
{plugin.packagename()} = callPackage ({{ {", ".join(sorted(call_args))} }}: {builder} {{
|
||
|
|
pname = "{plugin.slug}";
|
||
|
|
plugname = "{plugin.name}";
|
||
|
|
plugid = "{plugin.id}";
|
||
|
|
version = "{plugin.version}";
|
||
|
|
src = fetchurl {{
|
||
|
|
url = "{src_url}";
|
||
|
|
sha256 = "{sha}";
|
||
|
|
name = "{plugin.filename()}{src_ext}";
|
||
|
|
}};
|
||
|
|
buildInputs = [ {" ".join(build_inputs)} ];
|
||
|
|
packageRequires = [ {" ".join(requires)} ];
|
||
|
|
meta = {{
|
||
|
|
homepage = "{plugin.url or ""}";
|
||
|
|
license = {license};
|
||
|
|
description = ''
|
||
|
|
{plugin.description()}
|
||
|
|
'';
|
||
|
|
}};
|
||
|
|
}}) {{}};
|
||
|
|
"""
|
||
|
|
)
|
||
|
|
outfile.write("}\n")
|
||
|
|
|
||
|
|
|
||
|
|
def main():
|
||
|
|
parser = argparse.ArgumentParser()
|
||
|
|
parser.add_argument(
|
||
|
|
"-n", "--number", type=int, help="Limit the number of packages to write",
|
||
|
|
)
|
||
|
|
parser.add_argument(
|
||
|
|
"-o", "--out", type=str, help="File to write plugins to",
|
||
|
|
)
|
||
|
|
parser.add_argument(
|
||
|
|
"package",
|
||
|
|
metavar="PACKAGE",
|
||
|
|
type=str,
|
||
|
|
help="The Nixpkgs package name (inc. version)",
|
||
|
|
)
|
||
|
|
|
||
|
|
args = parser.parse_args()
|
||
|
|
|
||
|
|
build = Build(args.package)
|
||
|
|
plugins = list_plugins(build)
|
||
|
|
plugins.sort(key=lambda p: p.slug)
|
||
|
|
deduplicate(plugins)
|
||
|
|
|
||
|
|
if args.number:
|
||
|
|
plugins = plugins[: args.number]
|
||
|
|
|
||
|
|
print(f"Generating packages for {len(plugins)} plugins", file=sys.stderr)
|
||
|
|
if not args.out:
|
||
|
|
write_packages(sys.stdout, plugins, build)
|
||
|
|
else:
|
||
|
|
with open(args.out, "w") as f:
|
||
|
|
write_packages(f, plugins, build)
|
||
|
|
|
||
|
|
|
||
|
|
main()
|