Test mixed linux/src repos

See r-universe-org/help#521
r-universe-org · Nov 8, 2024 · 2cf6196 · 2cf6196
1 parent 3926af7
commit 2cf6196
Show file tree

Hide file tree

Showing 3 changed files with 60 additions and 33 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -4,7 +4,7 @@ RUN apk add --no-cache bash tini
 
 EXPOSE 3000
 
-RUN mkdir /app && cd /app && npm install [email protected].23
+RUN mkdir /app && cd /app && npm install [email protected].25
 
 WORKDIR /app/node_modules/cranlike
 

diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "cranlike",
-  "version": "0.22.23",
+  "version": "0.22.25",
   "type": "module",
   "description": "High-performance R package server",
   "author": "Jeroen Ooms <[email protected]>",

diff --git a/routes/repos.js b/routes/repos.js
@@ -32,7 +32,7 @@ function etagify(x){
   return 'W/"' +  x + '"';
 }
 
-function packages_index(query, format, req, res, next){
+function stream_to_dcf(cursor, format, req, res, next){
   if(format == 'rds'){
     return res.status(404).send("PACKAGES.rds format not supported for now");
   }
@@ -47,45 +47,44 @@ function packages_index(query, format, req, res, next){
     });
   }
 
-  // Preflight to revalidate cache.
+  // Get actual package data
+  var cursor = cursor.project(projection).sort({"Package" : 1});
+  if(!format){
+    cursor
+      .stream({transform: doc_to_dcf})
+      .pipe(res.type('text/plain'));
+  } else if(format == 'gz'){
+    cursor
+      .stream({transform: doc_to_dcf})
+      .pipe(zlib.createGzip())
+      .pipe(res.type('application/x-gzip'));
+  } else if(format == 'json'){
+    cursor
+      .stream({transform: doc_to_ndjson})
+      .pipe(res.type('text/plain'));
+  } else {
+    cursor.close();
+    next(createError(404, 'Unknown PACKAGES format: ' + format));
+  }
+}
+
+function packages_index(query, format, req, res, next){
+  // Try mitigate hammering. Cache for at least 10 sec, after that revalidate.
+  // This requires nginx to use proxy_cache_revalidate;
   packages.find(query).sort({"_id" : -1}).limit(1).project({"_id": 1}).next().then(function(doc){
     if(!doc){
       res.status(200).send();
-      return; //DONE!
+      return;
     }
-
-    // Try mitigate hammering. Cache for at least 10 sec, after that revalidate.
-    // This requires nginx to use proxy_cache_revalidate;
     var etag = etagify(doc['_id']);
     res.set('ETag', etag);
     res.set('Cache-Control', 'public, max-age=10, must-revalidate');
-
-    // Revalidate:
     if(etag === req.header('If-None-Match')){
       res.status(304).send();
-      return; //DONE!
-    }
-
-    // Get actual package data
-    var cursor = packages.find(query).project(projection).sort({"Package" : 1});
-    if(!format){
-      cursor
-        .stream({transform: doc_to_dcf})
-        .pipe(res.type('text/plain'));
-    } else if(format == 'gz'){
-      cursor
-        .stream({transform: doc_to_dcf})
-        .pipe(zlib.createGzip())
-        .pipe(res.type('application/x-gzip'));
-    } else if(format == 'json'){
-      cursor
-        .stream({transform: doc_to_ndjson})
-        .pipe(res.type('text/plain'));
-    } else {
-      cursor.close();
-      next(createError(404, 'Unknown PACKAGES format: ' + format));
+      return;
     }
-  }).catch(error_cb(400, next));
+    return stream_to_dcf(packages.find(query), format, req, res, next);
+  });
 }
 
 function html_index(query, res){
@@ -130,8 +129,9 @@ function count_by_built(user, type){
   .stream({transform: doc_to_ndjson});
 }
 
+//sort by type to get linux binary before src
 function query_stream_info(query){
-  return packages.findOne(query, {project: {_fileid: 1, Redirect: 1}}).then(function(docs){
+  return packages.findOne(query, {sort: {_type: -1}, project: {_fileid: 1, Redirect: 1}}).then(function(docs){
     if(!docs)
       throw 'Package not found for query: ' + JSON.stringify(query);
     var hash = docs._fileid;
@@ -249,6 +249,24 @@ router.get('/:user/bin/linux/:distro/:built/src/contrib/', function(req, res, ne
   packages_index(query, 'json', req, res, next);
 });
 
+/* Mix linux binaries with source packages */
+router.get('/:user/bin/mixed/:distro/:built/src/contrib/PACKAGES\.:ext?', function(req, res, next) {
+  var query = {_user: req.params.user, '$or': [
+    {_type: 'src'},
+    {_type: 'linux', '_distro': req.params.distro, 'Built.R' : {$regex: '^' + req.params.built}},
+  ]};
+  var cursor = packages.aggregate([
+    {$match: query},
+    {$sort: {_type: 1}},
+    {$group : {
+      _id : {'Package': '$Package'},
+      doc: { '$first': '$$ROOT' }
+    }},
+    {$replaceRoot: { newRoot: '$doc' }}
+  ]);
+  stream_to_dcf(cursor, req.params.ext, req, res, next);
+});
+
 router.get('/:user/bin/linux/:distro/:built', function(req, res, next) {
   res.redirect(req.path + '/src/contrib');
 });
@@ -306,6 +324,15 @@ router.get('/:user/bin/linux/:distro/:built/src/contrib/:pkg.tar.gz', function(r
   send_binary(query, req, res, next);
 });
 
+router.get('/:user/bin/mixed/:distro/:built/src/contrib/:pkg.tar.gz', function(req, res, next) {
+  var [pkg, version] = req.params.pkg.split("_");
+  var query = {_user: req.params.user, Package: pkg, Version: version, '$or': [
+    {_type: 'src'},
+    {_type: 'linux', '_distro': req.params.distro, 'Built.R' : {$regex: '^' + req.params.built}},
+  ]};
+  send_binary(query, req, res, next);
+});
+
 router.get('/:user/bin/emscripten/contrib/:built/:pkg.(tgz|data.gz)', function(req, res, next) {
   var [pkg, version] = req.params.pkg.split("_");
   var query = qf({_user: req.params.user, _type: 'wasm', 'Built.R' : {$regex: '^' + req.params.built},