CrawledUrl.scala

sealed trait CrawledUrl { val url: URl }
case class PendingURL extends CrawledUrl(url: URL)
case class InFlightURL extends CrawledUrl(url: URL)
case class VisitedURL extends CrawledUrl(url: URL)

object UrlSource2 {
    val urlStore: scala.collection.concurrent.Map[URL, CrawledUrl]

    def completeTask(doneUrl: URL): Unit = {
        urlStore.updateWith(doneUrl, VisitedURL(doneUrl.url))
    }

    def consumeTask(): Option[URL] = {
        urlStore.find {
            case url: PendingURL => url
        } map { foundUrl
            urlStore.updateWith(foundUrl, InFlightURL(doneUrl.url))
        }
    }
}