From c80a76c776d682e2743404fe62c5685e2a196b17 Mon Sep 17 00:00:00 2001 From: Christopher Hunt Date: Wed, 16 Sep 2015 19:27:55 +1000 Subject: [PATCH] Interpret a version from a ref Use the ref field of a web hook to determine a version number. The web hook conveys which branch should be updated. --- .travis.yml | 3 + README.md | 2 + app/controllers/Application.scala | 135 ++++++++++------- app/doc/DocRenderer.scala | 21 +-- app/doc/DocVersions.scala | 5 - app/modules/ConductRDocRendererModule.scala | 36 +++-- app/views/mainNav.scala.html | 2 +- build.sbt | 8 +- conf/routes | 3 +- project/plugins.sbt | 2 +- test/controllers/ApplicationSpec.scala | 151 ++++++++++++++++++-- test/doc/DocRendererSpec.scala | 22 ++- 12 files changed, 286 insertions(+), 104 deletions(-) create mode 100644 .travis.yml delete mode 100644 app/doc/DocVersions.scala diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..1b9cf0c --- /dev/null +++ b/.travis.yml @@ -0,0 +1,3 @@ +jdk: oraclejdk8 +language: scala +script: sbt test diff --git a/README.md b/README.md index 70d8c50..630800b 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # Project Documentation +[![Build Status](https://api.travis-ci.org/typesafehub/project-doc.png?branch=master)](https://travis-ci.org/typesafehub/project-doc) + A general purpose project documentation website. ## Setting up development environment diff --git a/app/controllers/Application.scala b/app/controllers/Application.scala index d8e7d04..42fde53 100644 --- a/app/controllers/Application.scala +++ b/app/controllers/Application.scala @@ -1,103 +1,106 @@ package controllers -import java.io.File import javax.crypto.Mac import javax.crypto.spec.SecretKeySpec import javax.inject.{Named, Inject} import akka.actor.ActorRef import akka.pattern.{AskTimeoutException, ask} -import doc.{DocVersions, DocRenderer} +import doc.DocRenderer import play.api.libs.MimeTypes import play.api.libs.concurrent.Execution.Implicits.defaultContext -import play.api.libs.iteratee.{Enumerator, Iteratee} +import play.api.libs.iteratee.Iteratee +import play.api.libs.json.{JsError, JsSuccess, Json, JsPath} import play.api.mvc._ import play.twirl.api.Html import settings.Settings +import scala.collection.mutable.ArrayBuffer import scala.concurrent.Future object Application { private[controllers] object MacBodyParser { - def apply(hmacHeader: String, secret: SecretKeySpec, algorithm: String) = - new MacBodyParser(hmacHeader, secret, algorithm) + def apply(hmacHeader: String, secret: SecretKeySpec, algorithm: String, maxBodySize: Int = 8192) = + new MacBodyParser(hmacHeader, secret, algorithm, maxBodySize) } private[controllers] class MacBodyParser( hmacHeader: String, secret: SecretKeySpec, - algorithm: String) extends BodyParser[Unit] { + algorithm: String, + maxBodySize: Int) extends BodyParser[Array[Byte]] { def hex2bytes(hex: String): Array[Byte] = hex.replaceAll("[^0-9A-Fa-f]", "").sliding(2, 2).toArray.map(Integer.parseInt(_, 16).toByte) - override def apply(request: RequestHeader): Iteratee[Array[Byte], Either[Result, Unit]] = { + override def apply(request: RequestHeader): Iteratee[Array[Byte], Either[Result, Array[Byte]]] = { val hexSignature = request.headers.get(hmacHeader).map(_.dropWhile(_ != '=').drop(1)).getOrElse("") val signature = hex2bytes(hexSignature) - Iteratee.fold[Array[Byte], Mac] { + Iteratee.fold[Array[Byte], (Mac, ArrayBuffer[Byte])] { val mac = Mac.getInstance(algorithm) mac.init(secret) - mac - } { (mac, bytes) => - mac.update(bytes) - mac + (mac, ArrayBuffer.empty) + } { + case ((mac, buffer), bytes) => + mac.update(bytes) + val newBuffer = if (buffer.length + bytes.length <= maxBodySize) buffer ++ bytes else buffer + (mac, newBuffer) }.map { - case _ if signature.isEmpty => Left(Results.BadRequest(s"No $hmacHeader header present")) - case mac if mac.doFinal().sameElements(signature) => Right(()) - case _ => Left(Results.Unauthorized("Bad signature")) + case _ if signature.isEmpty => Left(Results.BadRequest(s"No $hmacHeader header present")) + case (mac, buffer) if mac.doFinal().sameElements(signature) => Right(buffer.toArray) + case _ => Left(Results.Unauthorized("Bad signature")) } } } private def getDocRenderer( host: String, - docRenderers: Map[String, ActorRef], + pathVersion: String => Option[String], + docRenderers: Map[String, Map[String, ActorRef]], hostPrefixAliases: Map[String, String]): Option[ActorRef] = { + val hostPrefix = host.takeWhile(c => c != '.' && c != ':') - docRenderers.get(hostPrefix).orElse { - hostPrefixAliases.get(hostPrefix) match { - case Some(aliasedHostPrefix) => docRenderers.get(aliasedHostPrefix) - case None => None - } - } + + val resolvedHostPrefix = if (docRenderers.contains(hostPrefix)) Some(hostPrefix) else hostPrefixAliases.get(hostPrefix) + + for { + hp <- resolvedHostPrefix + dv <- docRenderers.get(hp) + pv <- pathVersion(hp) + dr <- dv.get(pv) + } yield dr } } class Application @Inject() ( - @Named("ConductRDocRenderer") conductrDocRenderer: ActorRef, + @Named("ConductRDocRenderer10") conductrDocRenderer10: ActorRef, + @Named("ConductRDocRenderer11") conductrDocRenderer11: ActorRef, settings: Settings) extends Controller { import Application._ - private final val MacAlgorithm = "HmacSHA1" - private final val GitHubSignature = "X-Hub-Signature" - - private val docRenderers = Map("conductr" -> conductrDocRenderer) - - private val secret = new SecretKeySpec(settings.play.crypto.secret.getBytes, MacAlgorithm) - def renderIndex = Action { Ok(views.html.conductr.index()) } - def renderDocsHome = - renderDocs("") + def renderDocsHome(version: String) = + renderDocs("", version) def renderResources(path: String, version: String) = - renderDocs(path) + renderDocs(path, version) - def renderDocs(path: String, version: String = DocVersions.Latest) = Action.async { request => + def renderDocs(path: String, version: String) = Action.async { request => request.headers.get(HOST) match { case Some(host) => - getDocRenderer(host, docRenderers, settings.application.hostAliases) match { + getDocRenderer(host, _ => Some(version), docRenderers, settings.application.hostAliases) match { case Some(docRenderer) => docRenderer .ask(DocRenderer.Render(path))(settings.doc.renderer.timeout) .map { case html: Html => Ok(html) case resource: DocRenderer.Resource => renderResource(resource, path) - case DocRenderer.Redirect(rp) => Redirect(routes.Application.renderDocs(rp, DocVersions.Latest)) + case DocRenderer.Redirect(rp, v) => Redirect(routes.Application.renderDocs(rp, v)) case DocRenderer.NotFound(rp) => NotFound(s"Cannot find $rp") case DocRenderer.NotReady => ServiceUnavailable("Initializing documentation. Please try again in a minute.") } @@ -112,26 +115,58 @@ class Application @Inject() ( } } - private def renderResource(resource: DocRenderer.Resource, path: String): Result = { - val fileName = path.drop(path.lastIndexOf('/') + 1) - Result(ResponseHeader(OK, Map[String, String]( - CONTENT_LENGTH -> resource.size.toString, - CONTENT_TYPE -> MimeTypes.forFileName(fileName).getOrElse(BINARY) - )), resource.content) - } - def update() = Action(MacBodyParser(GitHubSignature, secret, MacAlgorithm)) { request => request.headers.get(HOST) match { case Some(host) => - getDocRenderer(host, docRenderers, settings.application.hostAliases) match { - case Some(docRenderer) => - docRenderer ! DocRenderer.PropogateGetSite - Ok("Site update requested") - case None => - NotFound(s"Unknown project: $host") + Json.parse(request.body).validate[String](webhookRef) match { + case JsSuccess(ref, _) => + val branch = ref.reverse.takeWhile(_ != '/').reverse + + def branchToVersion(hostPrefix: String): Option[String] = + branchesToVersions.get(hostPrefix).flatMap(_.get(branch)) + + getDocRenderer(host, branchToVersion, docRenderers, settings.application.hostAliases) match { + case Some(docRenderer) => + docRenderer ! DocRenderer.PropogateGetSite + Ok("Site update requested") + case None => + Ok(s"Site update requested for Unknown project: $host - ignoring") + } + case e: JsError => + BadRequest(s"Cannot parse webhook: $e") } case None => NotFound("No host header") } } + + private final val MacAlgorithm = "HmacSHA1" + private final val GitHubSignature = "X-Hub-Signature" + + private val docRenderers = Map( + "conductr" -> Map( + "" -> conductrDocRenderer10, + "1.0.x" -> conductrDocRenderer10, + "1.1.x" -> conductrDocRenderer11 + ) + ) + + private val branchesToVersions = Map( + "conductr" -> Map( + "1.0" -> "1.0.x", + "master" -> "1.1.x" + ) + ) + + private val secret = new SecretKeySpec(settings.play.crypto.secret.getBytes, MacAlgorithm) + private val webhookRef = (JsPath \ "ref").read[String] + + private def renderResource(resource: DocRenderer.Resource, path: String): Result = { + val fileName = path.drop(path.lastIndexOf('/') + 1) + Result(ResponseHeader(OK, Map[String, String]( + CONTENT_LENGTH -> resource.size.toString, + CONTENT_TYPE -> MimeTypes.forFileName(fileName).getOrElse(BINARY) + )), resource.content) + } + } diff --git a/app/doc/DocRenderer.scala b/app/doc/DocRenderer.scala index c06a46f..9600a30 100644 --- a/app/doc/DocRenderer.scala +++ b/app/doc/DocRenderer.scala @@ -28,9 +28,9 @@ object DocRenderer { case class Render(path: String) /** - * Redirect to a relative documentation path + * Redirect to a relative documentation path given a known version */ - case class Redirect(path: String) + case class Redirect(path: String, version: String) /** * Path is not found @@ -68,12 +68,12 @@ object DocRenderer { def props( docArchive: URI, - removeRootSegment: Boolean, + removeRootSegmentOfArchive: Boolean, docRoot: Path, docUri: String, version: String, wsClient: WSClient): Props = - Props(new DocRenderer(docArchive, removeRootSegment, docRoot, docUri, version, wsClient)) + Props(new DocRenderer(docArchive, removeRootSegmentOfArchive, docRoot, docUri, version, wsClient)) private[doc] def unzip(input: Enumerator[Array[Byte]], removeRootSegment: Boolean)(implicit ec: ExecutionContext): Future[Path] = { val archive = Files.createTempFile(null, null) @@ -182,7 +182,7 @@ class DocRenderer( implicit val cluster = Cluster(context.system) override def preStart(): Unit = { - replicator ! Subscribe(SiteUpdateCounter, self) + replicator ! Subscribe(siteUpdateCounter, self) self ! GetSite } @@ -217,19 +217,22 @@ class DocRenderer( case PropogateGetSite => log.info(s"Notifying cluster of change for $docArchive") - replicator ! Update(SiteUpdateCounter, GCounter(), WriteLocal)(_ + 1) + replicator ! Update(siteUpdateCounter, GCounter(), WriteLocal)(_ + 1) - case Changed(SiteUpdateCounter, _: GCounter) => + case Changed(siteUpdateCounter, _: GCounter) => self ! GetSite } - + + private def siteUpdateCounter: String = + s"$SiteUpdateCounter/${self.path.name}/$version" + private def handleUnready: Receive = { case _ => sender() ! NotReady } private def handleRendering(repo: FilesystemRepository, mdRenderer: PlayDoc, toc: Html, toolbar: Html, cache: Cache[Html]): Receive = { case Render("") => - sender() ! Redirect(IndexPath) + sender() ! Redirect(IndexPath, version) case Render(path) if !path.contains(".") => cache(path) { diff --git a/app/doc/DocVersions.scala b/app/doc/DocVersions.scala deleted file mode 100644 index 5796365..0000000 --- a/app/doc/DocVersions.scala +++ /dev/null @@ -1,5 +0,0 @@ -package doc - -object DocVersions { - val Latest = "1.0.x" -} diff --git a/app/modules/ConductRDocRendererModule.scala b/app/modules/ConductRDocRendererModule.scala index dcd807c..324ea00 100644 --- a/app/modules/ConductRDocRendererModule.scala +++ b/app/modules/ConductRDocRendererModule.scala @@ -5,28 +5,45 @@ import java.nio.file.Paths import javax.inject.{Provider, Inject, Singleton} import akka.actor.{ActorRef, ActorSystem} -import doc.{DocVersions, DocRenderer} +import doc.DocRenderer import play.api.{Configuration, Environment} import play.api.inject.Module import play.api.libs.ws.WSClient object ConductRDocRendererModule { - @Singleton - class ConductRDocRendererProvider @Inject()(actorSystem: ActorSystem, wsClient: WSClient) + abstract class ConductRDocRendererProvider(actorSystem: ActorSystem, wsClient: WSClient, docArchive: URI, version: String) extends Provider[ActorRef] { private val renderer = actorSystem.actorOf(DocRenderer.props( - new URI("https://github.com/typesafehub/conductr-doc/archive/master.zip"), - removeRootSegment = true, + docArchive, + removeRootSegmentOfArchive = true, Paths.get("src/main/play-doc"), - controllers.routes.Application.renderDocsHome().url, - DocVersions.Latest, - wsClient), "conductr-doc-renderer") + controllers.routes.Application.renderDocsHome(version).url, + version, + wsClient), s"conductr-doc-renderer-$version") override def get = renderer } + + @Singleton + class ConductRDocRendererProvider10 @Inject()(actorSystem: ActorSystem, wsClient: WSClient) + extends ConductRDocRendererProvider( + actorSystem, + wsClient, + new URI("https://github.com/typesafehub/conductr-doc/archive/1.0.zip"), + "1.0.x" + ) + + @Singleton + class ConductRDocRendererProvider11 @Inject()(actorSystem: ActorSystem, wsClient: WSClient) + extends ConductRDocRendererProvider( + actorSystem, + wsClient, + new URI("https://github.com/typesafehub/conductr-doc/archive/master.zip"), + "1.1.x" + ) } class ConductRDocRendererModule extends Module { @@ -34,6 +51,7 @@ class ConductRDocRendererModule extends Module { def bindings(environment: Environment, configuration: Configuration) = Seq( - bind[ActorRef].qualifiedWith("ConductRDocRenderer").toProvider[ConductRDocRendererProvider] + bind[ActorRef].qualifiedWith("ConductRDocRenderer10").toProvider[ConductRDocRendererProvider10], + bind[ActorRef].qualifiedWith("ConductRDocRenderer11").toProvider[ConductRDocRendererProvider11] ) } \ No newline at end of file diff --git a/app/views/mainNav.scala.html b/app/views/mainNav.scala.html index 3454f3c..e097adf 100644 --- a/app/views/mainNav.scala.html +++ b/app/views/mainNav.scala.html @@ -1,6 +1,6 @@ @(showIcon: Boolean)