[med-svn] [r-bioc-annotationhub] 02/05: New upstream version 2.10.1
Andreas Tille
tille at debian.org
Thu Nov 9 09:16:04 UTC 2017
This is an automated email from the git hooks/post-receive script.
tille pushed a commit to branch master
in repository r-bioc-annotationhub.
commit dff0956b1b26efe7a177315c76958b63c615b84a
Author: Andreas Tille <tille at debian.org>
Date: Thu Nov 9 10:11:50 2017 +0100
New upstream version 2.10.1
---
DESCRIPTION | 17 +-
NAMESPACE | 4 +-
NEWS | 36 ++++
R/AnnotationHub-class.R | 42 +---
R/AnnotationHubResource-class.R | 21 ++
R/Hub-class.R | 49 ++++-
R/cache-utils.R | 2 +-
R/db-utils.R | 8 +-
R/readMetadataFromCsv.R | 83 ++++++++
R/sql-utils.R | 2 -
R/utilities.R | 18 +-
build/vignette.rds | Bin 287 -> 339 bytes
inst/doc/AnnotationHub-HOWTO.R | 4 +-
inst/doc/AnnotationHub-HOWTO.Rmd | 4 +-
inst/doc/AnnotationHub-HOWTO.html | 232 ++++++++++++++------
inst/doc/AnnotationHub.R | 42 ++--
inst/doc/AnnotationHub.html | 300 +++++++++++++++++---------
inst/doc/CreateAnAnnotationPackage.R | 25 +++
inst/doc/CreateAnAnnotationPackage.Rmd | 354 +++++++++++++++++++++++++++++++
inst/doc/CreateAnAnnotationPackage.html | 364 ++++++++++++++++++++++++++++++++
man/AnnotationHub-class.Rd | 21 +-
man/AnnotationHub-package.Rd | 10 +-
man/AnnotationHubResource-class.Rd | 39 +---
man/listResources.Rd | 54 +----
man/readMetadataFromCsv.Rd | 194 +++++++++++++++++
vignettes/AnnotationHub-HOWTO.Rmd | 4 +-
vignettes/CreateAnAnnotationPackage.Rmd | 354 +++++++++++++++++++++++++++++++
27 files changed, 1931 insertions(+), 352 deletions(-)
diff --git a/DESCRIPTION b/DESCRIPTION
index 9a1be67..6cb3530 100644
--- a/DESCRIPTION
+++ b/DESCRIPTION
@@ -1,7 +1,7 @@
Package: AnnotationHub
Type: Package
Title: Client to access AnnotationHub resources
-Version: 2.8.3
+Version: 2.10.1
Authors at R:
c(person("Martin", "Morgan", email="martin.morgan at roswellpark.org",
role="cre"),
@@ -21,23 +21,24 @@ Description: This package provides a client for the Bioconductor
access.
License: Artistic-2.0
Depends: BiocGenerics (>= 0.15.10)
-Imports: utils, methods, grDevices, RSQLite, BiocInstaller,
+Imports: utils, methods, grDevices, RSQLite, BiocInstaller, curl,
AnnotationDbi (>= 1.31.19), S4Vectors, interactiveDisplayBase,
httr, yaml
Suggests: IRanges, GenomicRanges, GenomeInfoDb, VariantAnnotation,
Rsamtools, rtracklayer, BiocStyle, knitr, AnnotationForge,
rBiopaxParser, RUnit, GenomicFeatures, MSnbase, mzR,
- Biostrings, SummarizedExperiment, ExperimentHub
+ Biostrings, SummarizedExperiment, ExperimentHub, gdsfmt
Enhances: AnnotationHubData
Collate: AnnotationHubOption.R AllGenerics.R Hub-class.R db-utils.R
- AnnotationHub-class.R AnnotationHubResource-class.R
- BEDResource-class.R ProteomicsResource-class.R
- EpigenomeResource-class.R EnsDbResource-class.R utilities.R
- sql-utils.R Hub-utils.R cache-utils.R zzz.R
+ readMetadataFromCsv.R AnnotationHub-class.R
+ AnnotationHubResource-class.R BEDResource-class.R
+ ProteomicsResource-class.R EpigenomeResource-class.R
+ EnsDbResource-class.R utilities.R sql-utils.R Hub-utils.R
+ cache-utils.R zzz.R
VignetteBuilder: knitr
NeedsCompilation: yes
Author: Martin Morgan [cre],
Marc Carlson [ctb],
Dan Tenenbaum [ctb],
Sonali Arora [ctb]
-Packaged: 2017-10-17 23:20:03 UTC; biocbuild
+Packaged: 2017-11-08 00:18:56 UTC; biocbuild
diff --git a/NAMESPACE b/NAMESPACE
index 3aa1421..babd02f 100644
--- a/NAMESPACE
+++ b/NAMESPACE
@@ -27,6 +27,7 @@ exportClasses(
AnnotationHub, AnnotationHubResource
)
+export(readMetadataFromCsv)
### - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
### Export S4 methods for generics not defined in AnnotationHub
###
@@ -53,8 +54,7 @@ exportMethods(
cache, "cache<-", query,
hubUrl, hubCache, hubDate,
snapshotDate, "snapshotDate<-",
- getHub, listResources, loadResources,
- package
+ getHub
)
### - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
diff --git a/NEWS b/NEWS
index 4468271..57b2b76 100644
--- a/NEWS
+++ b/NEWS
@@ -1,3 +1,39 @@
+CHANGES IN VERSION 2.10.1
+------------------------
+
+BUG FIXES
+
+ o Prompt for permission when downloading many (more than
+ AnnotationHubOption("MAX_DOWNLOADS")) resources.
+
+CHANGES IN VERSION 2.10.0
+------------------------
+
+NEW FEATURES
+
+ o AnnotationHub will now work offline utilizing argument 'localHub'; will
+ also use this option automatically if no internet connection is detected.
+
+ o Added new GDSResource class
+
+ o Added documentation for creating an AnnotationHub package
+
+MODIFICATIONS
+
+ o Modified tags vector when passed to display to improve speed of display
+ querying
+
+ o Moved readMetadataFromCsv from AnnotationHubData.
+
+ o Removed listResources and loadResource from AnnotationHub; not implemented
+ and only valid in ExperimentHub
+
+BUG FIXES
+
+ o Expose snapshot less than or equal to release date
+
+ o Force rebuild of index if index file corrput or out of date
+
CHANGES IN VERSION 2.8.0
------------------------
diff --git a/R/AnnotationHub-class.R b/R/AnnotationHub-class.R
index a93932e..b794e84 100644
--- a/R/AnnotationHub-class.R
+++ b/R/AnnotationHub-class.R
@@ -14,9 +14,15 @@ setClass("AnnotationHub", contains="Hub")
AnnotationHub <-
function(..., hub=getAnnotationHubOption("URL"),
cache=getAnnotationHubOption("CACHE"),
- proxy=getAnnotationHubOption("PROXY"))
+ proxy=getAnnotationHubOption("PROXY"),
+ localHub=FALSE)
{
- .Hub("AnnotationHub", hub, cache, proxy, ...)
+ connect <- curl::has_internet()
+ if (!connect && !localHub){
+ message("No internet connection using 'localHub=TRUE'")
+ localHub <- !connect
+ }
+ .Hub("AnnotationHub", hub, cache, proxy, localHub, ...)
}
### - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
@@ -87,35 +93,3 @@ setMethod("show", "AnnotationHub", function(object)
cat("# snapshotDate():", snapshotDate(object), "\n")
callNextMethod(object)
})
-
-### - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-### listResources and loadResources methods
-###
-
-.filterResources_AH <- function(package, filterBy=character()) {
- if (!is.character(filterBy))
- stop("'filterBy' must be a character vector")
- suppressMessages({ah <- AnnotationHub()})
- if (!package %in% unique(package(ah)))
- stop(paste0("no resources for package '", package,
- "' were found in AnnotationHub"))
-
- sub <- query(ah, package)
- if (length(filterBy))
- query(sub, filterBy)
- else
- sub
-}
-
-setMethod("listResources", "AnnotationHub",
- function(hub, package, filterBy=character()) {
- metadata <- .filterResources_AH(package, filterBy)
- mcols(metadata)$title
-})
-
-setMethod("loadResources", "AnnotationHub",
- function(hub, package, filterBy=character()) {
- metadata <- .filterResources_AH(package, filterBy)
- ah <- AnnotationHub()
- lapply(names(metadata), function(i) ah[[i]])
-})
diff --git a/R/AnnotationHubResource-class.R b/R/AnnotationHubResource-class.R
index 2ebf57c..efecf72 100644
--- a/R/AnnotationHubResource-class.R
+++ b/R/AnnotationHubResource-class.R
@@ -273,3 +273,24 @@ setMethod(".get1", "ExpressionSetResource",
.require("Biobase")
callNextMethod(x, ...)
})
+
+# GDS
+setClass("GDSResource", contains="AnnotationHubResource")
+
+setMethod(".get1", "GDSResource",
+ function(x, ...)
+{
+ .require("gdsfmt")
+ yy <- cache(getHub(x))
+ dat <- gdsfmt::openfn.gds(yy)
+})
+
+## H5FileResource
+setClass("H5FileResource", contains = "AnnotationHubResource")
+
+setMethod(".get1", "H5FileResource",
+ function(x, ...)
+{
+ .require("rhdf5")
+ cache(getHub(x))
+})
diff --git a/R/Hub-class.R b/R/Hub-class.R
index 6f744d2..0b4d027 100644
--- a/R/Hub-class.R
+++ b/R/Hub-class.R
@@ -19,19 +19,53 @@ setClass("Hub",
### Constructor for subclasses
###
-.Hub <- function(.class, url, cache, proxy, ...) {
+.Hub <- function(.class, url, cache, proxy, localHub=FALSE, ...) {
db_path <- .create_cache(cache, .class)
- db_path <- .db_get(db_path, url, proxy)
- db_date <- .restrictDateByVersion(db_path)
+ if (!localHub){
+ db_path <- .db_get(db_path, url, proxy)
+ tryCatch({
+ db_date <- .restrictDateByVersion(db_path)
+ }, error=function(err) {
+ stop("failed to connect",
+ "\n reason: ", conditionMessage(err),
+ "\n Consider rerunning with 'localHub=TRUE'")
+ })
+ } else {
+ if (!file.exists(db_path))
+ stop("Local database does not exist.",
+ "\n Repeat call with 'localHub=FALSE'")
+ dates <-.possibleDates(db_path)
+ db_date <- dates[length(dates)]
+ }
db_uid <- .db_uid0(db_path, db_date)
hub <- new(.class, cache=cache, hub=url, date=db_date,
.db_path=db_path, .db_uid=db_uid, ...)
+
message("snapshotDate(): ", snapshotDate(hub))
- index <- .db_create_index(hub)
- .db_index(hub) <- index
+
+ if (!localHub){
+ index <- .db_create_index(hub)
+ .db_index(hub) <- index
+ } else{
+ index <- file.path(dirname(db_path), "index.rds")
+ if (!file.exists(index))
+ stop("Index file not created.",
+ "\n Repeat call with 'localHub=FALSE'")
+ .db_index(hub) <- index
+ hub <- .subsethub(hub)
+ }
hub
}
+.subsethub <- function(hub){
+ myX = .named_cache_path(hub)
+ locFiles = setdiff(dir(hubCache(hub)),
+ c("annotationhub.sqlite3", "index.rds"))
+ dx = which(basename(myX) %in% locFiles)
+ files = names(myX[dx])
+ hub[files]
+}
+
### - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
### Validity
###
@@ -430,7 +464,10 @@ setMethod("show", "Hub", function(object)
#dropping column names
drops <-c("title", "taxonomyid", "sourceurl")
df <- df[,!(names(df) %in% drops), drop=FALSE]
-
+ # speeds up search by making character
+ df$tags <- vapply(unname(unclass(md$tags)), base::paste,
+ character(1), collapse=", ")
+
summaryMessage = capture.output(show(object))
serverOptions= list(
bSortClasses=TRUE,
diff --git a/R/cache-utils.R b/R/cache-utils.R
index 5668ef9..5775190 100644
--- a/R/cache-utils.R
+++ b/R/cache-utils.R
@@ -63,7 +63,7 @@
message("loading from cache ",
paste0(sQuote(cachepath), collapse="\n "))
}
- if (any(need) > max.downloads) {
+ if (sum(need) > max.downloads) {
if (!interactive()) {
txt <- sprintf("resources needed (%d) exceeds max.downloads (%d)",
sum(need), max.downloads)
diff --git a/R/db-utils.R b/R/db-utils.R
index 0526da3..24eaf76 100644
--- a/R/db-utils.R
+++ b/R/db-utils.R
@@ -128,8 +128,12 @@
.db_create_index <- function(x) {
fl <- .db_index_file(x)
- if (file.exists(fl) && (file.mtime(fl) > file.mtime(dbfile(x))))
- return(fl)
+
+ if (file.exists(fl)) {
+ if (file.mtime(fl) > file.mtime(dbfile(x)) &&
+ length(x) == length(readRDS(fl)))
+ return(fl)
+ }
tryCatch({
tbl <- .resource_table(x)
diff --git a/R/readMetadataFromCsv.R b/R/readMetadataFromCsv.R
new file mode 100644
index 0000000..7f7d29f
--- /dev/null
+++ b/R/readMetadataFromCsv.R
@@ -0,0 +1,83 @@
+### =========================================================================
+### readMetadataFromCsv()
+### -------------------------------------------------------------------------
+###
+
+## High level helper used to check metadata in 'Hub' packages.
+readMetadataFromCsv <- function(pathToPackage, fileName=character())
+{
+ if (!length(fileName))
+ fileName <- "metadata.csv"
+ path <- file.path(pathToPackage, "inst", "extdata")
+ meta <- read.csv(file.path(path, fileName), colClasses="character",
+ stringsAsFactors=FALSE)
+ mat <- rbind(c("Title", "character"),
+ c("Description", "character"),
+ c("BiocVersion", "character"),
+ c("Genome", "character"),
+ c("SourceType", "character"),
+ c("SourceUrl", "character"),
+ c("SourceVersion", "character"),
+ c("Species", "character"),
+ c("TaxonomyId", "integer"),
+ c("Coordinate_1_based", "logical"),
+ c("DataProvider", "character"),
+ c("Maintainer", "character"),
+ c("RDataClass", "character"),
+ c("DispatchClass", "character"),
+ c("RDataPath", "character"))
+
+ expected <- mat[,1]
+ missing <- !expected %in% names(meta)
+ if (any(missing))
+ stop("missing fields in metadata file ", fileName, ": ",
+ paste(expected[missing], collapse=", "))
+ extra<- !names(meta) %in% expected
+
+ ## All fields length 1
+ apply(meta, 1,
+ function(xx) {
+ valid <- sapply(xx, function(field) length(field) == 1L)
+ if (any(!valid))
+ stop("all fields in ", fileName, " must be a character ",
+ "string of length 1")
+ }
+
+ )
+ ## Populate required fields
+ missing <- is.na(nchar(meta$DataProvider))
+ if (any(missing)) {
+ meta$DataProvider[missing] <- "NA"
+ message("missing values for 'DataProvider set to 'NA''")
+ }
+ if (any(is.na(meta$Coordinate_1_based))) {
+ meta$Coordinate_1_based <- TRUE
+ message("missing values for 'Coordinate_1_based set to TRUE'")
+ } else {
+ meta$Coordinate_1_based <- as.logical(meta$Coordinate_1_based)
+ }
+ ## Enforce data type
+ meta$TaxonomyId <- as.integer(meta$TaxonomyId)
+ meta$BiocVersion <- package_version(meta$BiocVersion)
+
+ ## Location_Prefix not specified -> data in S3
+ if (all(is.null(Location_Prefix <- meta$Location_Prefix))) {
+ meta$Location_Prefix <- 'http://s3.amazonaws.com/annotationhub/'
+ ## Location_Prefix specified -> data at other location
+ }
+
+ if(all(
+ (meta$Location_Prefix == 'http://s3.amazonaws.com/annotationhub/') ||
+ (meta$Location_Prefix == 'http://s3.amazonaws.com/experimenthub/'))
+ ){
+ package <- basename(pathToPackage)
+ test <- vapply(meta$RDataPath, startsWith, logical(1), package)
+ if (!all(test)){
+ stop("RDataPath must start with package name: ", package)
+ }
+ }
+
+ ## Real time assignments
+ meta$RDataDateAdded <- rep(Sys.time(), nrow(meta))
+ meta
+}
diff --git a/R/sql-utils.R b/R/sql-utils.R
index e007e17..e1369db 100644
--- a/R/sql-utils.R
+++ b/R/sql-utils.R
@@ -228,8 +228,6 @@
}
## This is used by cache to get the rDataPath ID for a resource
-## I think this should say to select 'id' as id to extract the rdatapathID
-## (instead of the resource_id)
.datapathIds <- function(x)
{
query <- sprintf(
diff --git a/R/utilities.R b/R/utilities.R
index b7291d7..e88ffd8 100644
--- a/R/utilities.R
+++ b/R/utilities.R
@@ -1,21 +1,23 @@
.require <-
function(pkg)
{
+ result <- TRUE
if (length(grep(sprintf("package:%s", pkg), search())) != 0L)
- return()
+ return(invisible(result))
message("require(", dQuote(pkg), ")")
+
handler <- function(err) {
msg <- sprintf("require(%s) failed: %s", dQuote(pkg),
conditionMessage(err))
stop(msg)
}
- tryCatch({
- suppressPackageStartupMessages({
- require(pkg, quietly=TRUE, character.only=TRUE)
- })
- }, ## error first, so warnings-converted-to-error are not handled
- ## again
- error=handler, warning=handler)
+
+ result <- tryCatch(suppressPackageStartupMessages(
+ require(pkg, quietly=TRUE, character.only=TRUE)
+ ), error=handler)
+ if (!result)
+ handler(simpleError("use biocLite() to install package?"))
+ invisible(result)
}
.ask <- function(txt, values) {
diff --git a/build/vignette.rds b/build/vignette.rds
index 6e1b8bc..1afa08c 100644
Binary files a/build/vignette.rds and b/build/vignette.rds differ
diff --git a/inst/doc/AnnotationHub-HOWTO.R b/inst/doc/AnnotationHub-HOWTO.R
index f382f5a..ea029a1 100644
--- a/inst/doc/AnnotationHub-HOWTO.R
+++ b/inst/doc/AnnotationHub-HOWTO.R
@@ -128,8 +128,8 @@ genome(gr38) <- "hg38"
gr38
## ----echo=FALSE, results='hide', message=FALSE----------------------------------------------------
-query(ah, c("GRCh37", "dbSNP", "VCF" ))
-vcf <- ah[['AH50424']]
+query(ah, c("GRCh38", "dbSNP", "VCF" ))
+vcf <- ah[['AH57960']]
## ----message=FALSE--------------------------------------------------------------------------------
variants <- readVcf(vcf, genome="hg19")
diff --git a/inst/doc/AnnotationHub-HOWTO.Rmd b/inst/doc/AnnotationHub-HOWTO.Rmd
index 953199a..2bbec22 100644
--- a/inst/doc/AnnotationHub-HOWTO.Rmd
+++ b/inst/doc/AnnotationHub-HOWTO.Rmd
@@ -307,8 +307,8 @@ evidence of medical interest. This information is available at
Query the dbDNP files in the hub:
```{r echo=FALSE, results='hide', message=FALSE}
-query(ah, c("GRCh37", "dbSNP", "VCF" ))
-vcf <- ah[['AH50424']]
+query(ah, c("GRCh38", "dbSNP", "VCF" ))
+vcf <- ah[['AH57960']]
```
This returns a _VcfFile_ which can be read in using `r
Biocpkg("VariantAnnotation")`; because VCF files can be large, `readVcf()`
diff --git a/inst/doc/AnnotationHub-HOWTO.html b/inst/doc/AnnotationHub-HOWTO.html
index 32e2589..1d25bb9 100644
--- a/inst/doc/AnnotationHub-HOWTO.html
+++ b/inst/doc/AnnotationHub-HOWTO.html
@@ -4,22 +4,28 @@
<head>
-<meta charset="utf-8">
+<meta charset="utf-8" />
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
<meta name="generator" content="pandoc" />
+
<title>AnnotationHub How-To’s</title>
+<script src="data:application/x-javascript;base64,LyohIGpRdWVyeSB2MS4xMS4zIHwgKGMpIDIwMDUsIDIwMTUgalF1ZXJ5IEZvdW5kYXRpb24sIEluYy4gfCBqcXVlcnkub3JnL2xpY2Vuc2UgKi8KIWZ1bmN0aW9uKGEsYil7Im9iamVjdCI9PXR5cGVvZiBtb2R1bGUmJiJvYmplY3QiPT10eXBlb2YgbW9kdWxlLmV4cG9ydHM/bW9kdWxlLmV4cG9ydHM9YS5kb2N1bWVudD9iKGEsITApOmZ1bmN0aW9uKGEpe2lmKCFhLmRvY3VtZW50KXRocm93IG5ldyBFcnJvcigialF1ZXJ5IHJlcXVpcmVzIGEgd2luZG93IHdpdGggYSBkb2N1bWVudCIpO3JldHVybiBiKGEpfTpiKGEpfSgidW5kZWZpbmVkIiE9dHlwZW9mIHdpbmRvdz93aW5kb3c6dG [...]
+<meta name="viewport" content="width=device-width, initial-scale=1" />
+<link href="data:text/css;charset=utf-8,html%7Bfont%2Dfamily%3Asans%2Dserif%3B%2Dwebkit%2Dtext%2Dsize%2Dadjust%3A100%25%3B%2Dms%2Dtext%2Dsize%2Dadjust%3A100%25%7Dbody%7Bmargin%3A0%7Darticle%2Caside%2Cdetails%2Cfigcaption%2Cfigure%2Cfooter%2Cheader%2Chgroup%2Cmain%2Cmenu%2Cnav%2Csection%2Csummary%7Bdisplay%3Ablock%7Daudio%2Ccanvas%2Cprogress%2Cvideo%7Bdisplay%3Ainline%2Dblock%3Bvertical%2Dalign%3Abaseline%7Daudio%3Anot%28%5Bcontrols%5D%29%7Bdisplay%3Anone%3Bheight%3A0%7D%5Bhidden%5D%2Ctem [...]
+<script src="data:application/x-javascript;base64,LyohCiAqIEJvb3RzdHJhcCB2My4zLjUgKGh0dHA6Ly9nZXRib290c3RyYXAuY29tKQogKiBDb3B5cmlnaHQgMjAxMS0yMDE1IFR3aXR0ZXIsIEluYy4KICogTGljZW5zZWQgdW5kZXIgdGhlIE1JVCBsaWNlbnNlCiAqLwppZigidW5kZWZpbmVkIj09dHlwZW9mIGpRdWVyeSl0aHJvdyBuZXcgRXJyb3IoIkJvb3RzdHJhcCdzIEphdmFTY3JpcHQgcmVxdWlyZXMgalF1ZXJ5Iik7K2Z1bmN0aW9uKGEpeyJ1c2Ugc3RyaWN0Ijt2YXIgYj1hLmZuLmpxdWVyeS5zcGxpdCgiICIpWzBdLnNwbGl0KCIuIik7aWYoYlswXTwyJiZiWzFdPDl8fDE9PWJbMF0mJjk9PWJbMV0mJmJbMl08MSl0aHJvdy [...]
+<script src="data:application/x-javascript;base64,LyoqCiogQHByZXNlcnZlIEhUTUw1IFNoaXYgMy43LjIgfCBAYWZhcmthcyBAamRhbHRvbiBAam9uX25lYWwgQHJlbSB8IE1JVC9HUEwyIExpY2Vuc2VkCiovCi8vIE9ubHkgcnVuIHRoaXMgY29kZSBpbiBJRSA4CmlmICghIXdpbmRvdy5uYXZpZ2F0b3IudXNlckFnZW50Lm1hdGNoKCJNU0lFIDgiKSkgewohZnVuY3Rpb24oYSxiKXtmdW5jdGlvbiBjKGEsYil7dmFyIGM9YS5jcmVhdGVFbGVtZW50KCJwIiksZD1hLmdldEVsZW1lbnRzQnlUYWdOYW1lKCJoZWFkIilbMF18fGEuZG9jdW1lbnRFbGVtZW50O3JldHVybiBjLmlubmVySFRNTD0ieDxzdHlsZT4iK2IrIjwvc3R5bGU+IixkLm [...]
+<script src="data:application/x-javascript;base64,LyohIFJlc3BvbmQuanMgdjEuNC4yOiBtaW4vbWF4LXdpZHRoIG1lZGlhIHF1ZXJ5IHBvbHlmaWxsICogQ29weXJpZ2h0IDIwMTMgU2NvdHQgSmVobAogKiBMaWNlbnNlZCB1bmRlciBodHRwczovL2dpdGh1Yi5jb20vc2NvdHRqZWhsL1Jlc3BvbmQvYmxvYi9tYXN0ZXIvTElDRU5TRS1NSVQKICogICovCgovLyBPbmx5IHJ1biB0aGlzIGNvZGUgaW4gSUUgOAppZiAoISF3aW5kb3cubmF2aWdhdG9yLnVzZXJBZ2VudC5tYXRjaCgiTVNJRSA4IikpIHsKIWZ1bmN0aW9uKGEpeyJ1c2Ugc3RyaWN0IjthLm1hdGNoTWVkaWE9YS5tYXRjaE1lZGlhfHxmdW5jdGlvbihhKXt2YXIgYixjPWEuZG [...]
+<script src="data:application/x-javascript;base64,CgovKioKICogalF1ZXJ5IFBsdWdpbjogU3RpY2t5IFRhYnMKICoKICogQGF1dGhvciBBaWRhbiBMaXN0ZXIgPGFpZGFuQHBocC5uZXQ+CiAqIGFkYXB0ZWQgYnkgUnViZW4gQXJzbGFuIHRvIGFjdGl2YXRlIHBhcmVudCB0YWJzIHRvbwogKiBodHRwOi8vd3d3LmFpZGFubGlzdGVyLmNvbS8yMDE0LzAzL3BlcnNpc3RpbmctdGhlLXRhYi1zdGF0ZS1pbi1ib290c3RyYXAvCiAqLwooZnVuY3Rpb24oJCkgewogICJ1c2Ugc3RyaWN0IjsKICAkLmZuLnJtYXJrZG93blN0aWNreVRhYnMgPSBmdW5jdGlvbigpIHsKICAgIHZhciBjb250ZXh0ID0gdGhpczsKICAgIC8vIFNob3cgdGhlIHRhYi [...]
<link href="data:text/css;charset=utf-8,pre%20%2Eoperator%2C%0Apre%20%2Eparen%20%7B%0Acolor%3A%20rgb%28104%2C%20118%2C%20135%29%0A%7D%0Apre%20%2Eliteral%20%7B%0Acolor%3A%20%23990073%0A%7D%0Apre%20%2Enumber%20%7B%0Acolor%3A%20%23099%3B%0A%7D%0Apre%20%2Ecomment%20%7B%0Acolor%3A%20%23998%3B%0Afont%2Dstyle%3A%20italic%0A%7D%0Apre%20%2Ekeyword%20%7B%0Acolor%3A%20%23900%3B%0Afont%2Dweight%3A%20bold%0A%7D%0Apre%20%2Eidentifier%20%7B%0Acolor%3A%20rgb%280%2C%200%2C%200%29%3B%0A%7D%0Apre%20%2Estri [...]
<script src="data:application/x-javascript;base64,dmFyIGhsanM9bmV3IGZ1bmN0aW9uKCl7ZnVuY3Rpb24gbShwKXtyZXR1cm4gcC5yZXBsYWNlKC8mL2dtLCImYW1wOyIpLnJlcGxhY2UoLzwvZ20sIiZsdDsiKX1mdW5jdGlvbiBmKHIscSxwKXtyZXR1cm4gUmVnRXhwKHEsIm0iKyhyLmNJPyJpIjoiIikrKHA/ImciOiIiKSl9ZnVuY3Rpb24gYihyKXtmb3IodmFyIHA9MDtwPHIuY2hpbGROb2Rlcy5sZW5ndGg7cCsrKXt2YXIgcT1yLmNoaWxkTm9kZXNbcF07aWYocS5ub2RlTmFtZT09IkNPREUiKXtyZXR1cm4gcX1pZighKHEubm9kZVR5cGU9PTMmJnEubm9kZVZhbHVlLm1hdGNoKC9ccysvKSkpe2JyZWFrfX19ZnVuY3Rpb24gaCh0LH [...]
<style type="text/css">code{white-space: pre;}</style>
<style type="text/css">
- pre:not([class]) {
- background-color: white;
- }
+
</style>
<script type="text/javascript">
if (window.hljs && document.readyState && document.readyState === "complete") {
@@ -30,24 +36,84 @@ if (window.hljs && document.readyState && document.readyState === "complete") {
</script>
-<link href="data:text/css;charset=utf-8,body%2C%20td%20%7B%0Afont%2Dfamily%3A%20sans%2Dserif%3B%0Abackground%2Dcolor%3A%20white%3B%0Afont%2Dsize%3A%2013px%3B%0A%7D%0Abody%20%7B%0Amax%2Dwidth%3A%20800px%3B%0Amargin%3A%200%20auto%3B%0Apadding%3A%201em%201em%202em%3B%0Aline%2Dheight%3A%2020px%3B%0A%7D%0A%0Adiv%23TOC%20li%20%7B%0Alist%2Dstyle%3Anone%3B%0Abackground%2Dimage%3Anone%3B%0Abackground%2Drepeat%3Anone%3B%0Abackground%2Dposition%3A0%3B%0A%7D%0A%0Ap%2C%20pre%20%7B%20margin%3A%200em%2 [...]
-<script type="text/javascript">
-document.addEventListener("DOMContentLoaded", function() {
- var links = document.links;
- for (var i = 0, linksLength = links.length; i < linksLength; i++)
- if(links[i].hostname != window.location.hostname)
- links[i].target = '_blank';
-});
-</script>
+<style type="text/css">
+h1 {
+ font-size: 34px;
+}
+h1.title {
+ font-size: 38px;
+}
+h2 {
+ font-size: 30px;
+}
+h3 {
+ font-size: 24px;
+}
+h4 {
+ font-size: 18px;
+}
+h5 {
+ font-size: 16px;
+}
+h6 {
+ font-size: 12px;
+}
+.table th:not([align]) {
+ text-align: left;
+}
+</style>
+
+<link href="data:text/css;charset=utf-8,body%20%7B%0Amargin%3A%200px%20auto%3B%0Amax%2Dwidth%3A%201134px%3B%0Afont%2Dfamily%3A%20sans%2Dserif%3B%0Afont%2Dsize%3A%2010pt%3B%0A%7D%0A%0Adiv%23TOC%20ul%20%7B%0Apadding%3A%200px%200px%200px%2045px%3B%0Alist%2Dstyle%3A%20none%3B%0Abackground%2Dimage%3A%20none%3B%0Abackground%2Drepeat%3A%20none%3B%0Abackground%2Dposition%3A%200%3B%0Afont%2Dsize%3A%2010pt%3B%0Afont%2Dfamily%3A%20Helvetica%2C%20Arial%2C%20sans%2Dserif%3B%0A%7D%0Adiv%23TOC%20%3E%20 [...]
</head>
<body>
+<style type="text/css">
+.main-container {
+ max-width: 828px;
+ margin-left: auto;
+ margin-right: auto;
+}
+
+img {
+ max-width:100%;
+ height: auto;
+}
+.tabbed-pane {
+ padding-top: 12px;
+}
+button.code-folding-btn:focus {
+ outline: none;
+}
+</style>
+
+
+
+<div class="container-fluid main-container">
+
+<!-- tabsets -->
+<script>
+$(document).ready(function () {
+ window.buildTabsets("TOC");
+});
+</script>
+
+<!-- code folding -->
+
+
+
+
+
+
+<div class="fluid-row" id="header">
+
+
+
+<h1 class="title toc-ignore">AnnotationHub How-To’s</h1>
+<h4 class="date"><em>7 November 2017</em></h4>
-<div id="header">
-<h1 class="title">AnnotationHub How-To’s</h1>
</div>
<h1>Contents</h1>
@@ -77,7 +143,7 @@ document.addEventListener("DOMContentLoaded", function() {
links[i].target = '_blank';
});
</script>
-<p><strong>Package</strong>: <em><a href="http://bioconductor.org/packages/AnnotationHub">AnnotationHub</a></em><br /> <strong>Authors</strong>: Martin Morgan [cre], Marc Carlson [ctb], Dan Tenenbaum [ctb], Sonali Arora [ctb]<br /> <strong>Modified</strong>: Sun Jun 28 10:41:23 2015<br /> <strong>Compiled</strong>: Tue Oct 17 19:18:58 2017</p>
+<p><strong>Package</strong>: <em><a href="http://bioconductor.org/packages/AnnotationHub">AnnotationHub</a></em><br /> <strong>Authors</strong>: Martin Morgan [cre], Marc Carlson [ctb], Dan Tenenbaum [ctb], Sonali Arora [ctb]<br /> <strong>Modified</strong>: Sun Jun 28 10:41:23 2015<br /> <strong>Compiled</strong>: Tue Nov 7 19:17:44 2017</p>
<div id="accessing-genome-scale-data" class="section level1">
<h1><span class="header-section-number">1</span> Accessing Genome-Scale Data</h1>
<div id="non-model-organism-gene-annotations" class="section level2">
@@ -85,31 +151,31 @@ document.addEventListener("DOMContentLoaded", function() {
<p><em>Bioconductor</em> offers pre-built <code>org.*</code> annotation packages for model organisms, with their use described in the <a href="http://bioconductor.org/help/workflows/annotation/Annotation_Resources/#OrgDb">OrgDb</a> section of the Annotation work flow. Here we discover available <code>OrgDb</code> objects for less-model organisms</p>
<pre class="r"><code>library(AnnotationHub)
ah <- AnnotationHub()</code></pre>
-<pre><code>## snapshotDate(): 2017-04-25</code></pre>
+<pre><code>## snapshotDate(): 2017-10-27</code></pre>
<pre class="r"><code>query(ah, "OrgDb")</code></pre>
-<pre><code>## AnnotationHub with 940 records
-## # snapshotDate(): 2017-04-25
+<pre><code>## AnnotationHub with 1019 records
+## # snapshotDate(): 2017-10-27
## # $dataprovider: ftp://ftp.ncbi.nlm.nih.gov/gene/DATA/
-## # $species: Escherichia coli, Acanthamoeba castellanii_str._Neff, Acanthisitta chloris, Acetobac...
+## # $species: Escherichia coli, 'Caballeronia concitans', 'Chlorella vulgaris'_C-169, 'Frankia cas...
## # $rdataclass: OrgDb
## # additional mcols(): taxonomyid, genome, description, coordinate_1_based, maintainer,
## # rdatadateadded, preparerclass, tags, rdatapath, sourceurl, sourcetype
-## # retrieve records with, e.g., 'object[["AH53757"]]'
+## # retrieve records with, e.g., 'object[["AH57964"]]'
##
-## title
-## AH53757 | org.Ag.eg.db.sqlite
-## AH53758 | org.At.tair.db.sqlite
-## AH53759 | org.Bt.eg.db.sqlite
-## AH53760 | org.Cf.eg.db.sqlite
-## AH53761 | org.Gg.eg.db.sqlite
-## ... ...
-## AH56649 | org.Thermoplasmatales_archaeon_BRNA1.eg.sqlite
-## AH56650 | org.Ignicoccus_hospitalis_KIN4|I.eg.sqlite
-## AH56651 | org.Desulfurococcus_amylolyticus_DSM_16532.eg.sqlite
-## AH56652 | org.Pandoravirus_dulcis.eg.sqlite
-## AH56653 | org.Methanocaldococcus_infernus_ME.eg.sqlite</code></pre>
+## title
+## AH57964 | org.Ag.eg.db.sqlite
+## AH57965 | org.At.tair.db.sqlite
+## AH57966 | org.Bt.eg.db.sqlite
+## AH57967 | org.Cf.eg.db.sqlite
+## AH57968 | org.Gg.eg.db.sqlite
+## ... ...
+## AH58988 | org.Flavobacterium_piscicida.eg.sqlite
+## AH58989 | org.Bacteroides_fragilis_YCH46.eg.sqlite
+## AH58990 | org.Pseudomonas_mendocina_ymp.eg.sqlite
+## AH58991 | org.Salmonella_enterica_subsp._enterica_serovar_Typhimurium_str._LT2.eg.sqlite
+## AH58992 | org.Acinetobacter_baumannii.eg.sqlite</code></pre>
<pre class="r"><code>orgdb <- query(ah, "OrgDb")[[1]] </code></pre>
-<pre><code>## loading from cache '/home/biocbuild//.AnnotationHub/60495'</code></pre>
+<pre><code>## loading from cache '/home/biocbuild//.AnnotationHub/64710'</code></pre>
<p>The object returned by AnnotationHub is directly usable with the <code>select()</code> interface, e.g., to discover the available keytypes for querying the object, the columns that these keytypes can map to, and finally selecting the SYMBOL and GENENAME corresponding to the first 6 ENTREZIDs</p>
<pre class="r"><code>keytypes(orgdb)</code></pre>
<pre><code>## [1] "ACCNUM" "ENSEMBL" "ENSEMBLPROT" "ENSEMBLTRANS" "ENTREZID" "ENZYME"
@@ -144,12 +210,12 @@ if (file.exists(filename))
<p><em><a href="http://bioconductor.org/packages/AnnotationHub">AnnotationHub</a></em> reduces this task to just a few lines of <em>R</em> code</p>
<pre class="r"><code>library(AnnotationHub)
ah = AnnotationHub()</code></pre>
-<pre><code>## snapshotDate(): 2017-04-25</code></pre>
+<pre><code>## snapshotDate(): 2017-10-27</code></pre>
<pre class="r"><code>epiFiles <- query(ah, "EpigenomeRoadMap")</code></pre>
<p>A look at the value returned by <code>epiFiles</code> shows us that 18248 roadmap resources are available via <em><a href="http://bioconductor.org/packages/AnnotationHub">AnnotationHub</a></em>. Additional information about the files is also available, e.g., where the files came from (dataprovider), genome, species, sourceurl, sourcetypes.</p>
<pre class="r"><code>epiFiles</code></pre>
<pre><code>## AnnotationHub with 18248 records
-## # snapshotDate(): 2017-04-25
+## # snapshotDate(): 2017-10-27
## # $dataprovider: BroadInstitute
## # $species: Homo sapiens
## # $rdataclass: BigWigFile, GRanges, data.frame
@@ -234,7 +300,7 @@ ah = AnnotationHub()</code></pre>
<pre class="r"><code>metadata.tab <- query(ah , c("EpigenomeRoadMap", "Metadata"))
metadata.tab</code></pre>
<pre><code>## AnnotationHub with 1 record
-## # snapshotDate(): 2017-04-25
+## # snapshotDate(): 2017-10-27
## # names(): AH41830
## # $dataprovider: BroadInstitute
## # $species: Homo sapiens
@@ -272,7 +338,7 @@ metadata.tab</code></pre>
<pre class="r"><code>E126 <- query(ah , c("EpigenomeRoadMap", "E126", "H3K4ME2"))
E126</code></pre>
<pre><code>## AnnotationHub with 6 records
-## # snapshotDate(): 2017-04-25
+## # snapshotDate(): 2017-10-27
## # $dataprovider: BroadInstitute
## # $species: Homo sapiens
## # $rdataclass: BigWigFile, GRanges
@@ -336,7 +402,7 @@ E126</code></pre>
<p><em>AnnotationHub</em> provides an easy way to work with gene models published by Ensembl. Let’s see what Ensembl’s Release-80 has in terms of data for pufferfish, <em>Takifugu rubripes</em>.</p>
<pre class="r"><code>query(ah, c("Takifugu", "release-80"))</code></pre>
<pre><code>## AnnotationHub with 7 records
-## # snapshotDate(): 2017-04-25
+## # snapshotDate(): 2017-10-27
## # $dataprovider: Ensembl
## # $species: Takifugu rubripes
## # $rdataclass: FaFile, GRanges
@@ -397,6 +463,8 @@ gtf_subset <- gtf[seqnames(gtf) %in% keep]</code></pre>
<p>It is trivial to make a TxDb instance of this subset (or of the entire gtf)</p>
<pre class="r"><code>library(GenomicFeatures) # for makeTxDbFromGRanges
txdb <- makeTxDbFromGRanges(gtf_subset)</code></pre>
+<pre><code>## Warning in .get_cds_IDX(type, phase): The "phase" metadata column contains non-NA values for features of type stop_codon. This
+## information was ignored.</code></pre>
<p>and to use that in conjunction with the DNA sequences, e.g., to find exon sequences of all annotated genes.</p>
<pre class="r"><code>library(Rsamtools) # for getSeq,FaFile-method
exons <- exons(txdb)
@@ -426,7 +494,7 @@ length(exons)</code></pre>
<pre class="r"><code>chainfiles <- query(ah , c("hg38", "hg19", "chainfile"))
chainfiles</code></pre>
<pre><code>## AnnotationHub with 2 records
-## # snapshotDate(): 2017-04-25
+## # snapshotDate(): 2017-10-27
## # $dataprovider: UCSC
## # $species: Homo sapiens
## # $rdataclass: ChainFile
@@ -666,8 +734,8 @@ overlap[idx]</code></pre>
## Running under: Ubuntu 16.04.3 LTS
##
## Matrix products: default
-## BLAS: /home/biocbuild/bbs-3.5-bioc/R/lib/libRblas.so
-## LAPACK: /home/biocbuild/bbs-3.5-bioc/R/lib/libRlapack.so
+## BLAS: /home/biocbuild/bbs-3.6-bioc/R/lib/libRblas.so
+## LAPACK: /home/biocbuild/bbs-3.6-bioc/R/lib/libRlapack.so
##
## locale:
## [1] LC_CTYPE=en_US.UTF-8 LC_NUMERIC=C LC_TIME=en_US.UTF-8
@@ -679,37 +747,67 @@ overlap[idx]</code></pre>
## [1] stats4 parallel stats graphics grDevices utils datasets methods base
##
## other attached packages:
-## [1] BSgenome.Hsapiens.UCSC.hg19_1.4.0 BSgenome_1.44.2
-## [3] rtracklayer_1.36.6 VariantAnnotation_1.22.3
-## [5] SummarizedExperiment_1.6.5 DelayedArray_0.2.7
-## [7] matrixStats_0.52.2 Rsamtools_1.28.0
-## [9] Biostrings_2.44.2 XVector_0.16.0
-## [11] GenomicFeatures_1.28.5 AnnotationDbi_1.38.2
-## [13] Biobase_2.36.2 GenomicRanges_1.28.6
-## [15] GenomeInfoDb_1.12.3 IRanges_2.10.5
-## [17] S4Vectors_0.14.7 AnnotationHub_2.8.3
-## [19] BiocGenerics_0.22.1 BiocStyle_2.4.1
+## [1] BSgenome.Hsapiens.UCSC.hg19_1.4.0 BSgenome_1.46.0
+## [3] rtracklayer_1.38.0 VariantAnnotation_1.24.1
+## [5] SummarizedExperiment_1.8.0 DelayedArray_0.4.1
+## [7] matrixStats_0.52.2 Rsamtools_1.30.0
+## [9] Biostrings_2.46.0 XVector_0.18.0
+## [11] GenomicFeatures_1.30.0 AnnotationDbi_1.40.0
+## [13] Biobase_2.38.0 GenomicRanges_1.30.0
+## [15] GenomeInfoDb_1.14.0 IRanges_2.12.0
+## [17] S4Vectors_0.16.0 AnnotationHub_2.10.1
+## [19] BiocGenerics_0.24.0 BiocStyle_2.6.0
##
## loaded via a namespace (and not attached):
-## [1] lattice_0.20-35 htmltools_0.3.6 yaml_2.1.14
-## [4] interactiveDisplayBase_1.14.0 blob_1.1.0 XML_3.98-1.9
-## [7] rlang_0.1.2 DBI_0.7 BiocParallel_1.10.1
-## [10] bit64_0.9-7 GenomeInfoDbData_0.99.0 stringr_1.2.0
-## [13] zlibbioc_1.22.0 memoise_1.1.0 evaluate_0.10.1
-## [16] knitr_1.17 biomaRt_2.32.1 httpuv_1.3.5
-## [19] BiocInstaller_1.26.1 curl_3.0 Rcpp_0.12.13
-## [22] xtable_1.8-2 backports_1.1.1 mime_0.5
-## [25] bit_1.1-12 digest_0.6.12 stringi_1.1.5
-## [28] shiny_1.0.5 rprojroot_1.2 grid_3.4.2
-## [31] tools_3.4.2 bitops_1.0-6 magrittr_1.5
-## [34] RCurl_1.95-4.8 tibble_1.3.4 RSQLite_2.0
-## [37] pkgconfig_2.0.1 Matrix_1.2-11 rmarkdown_1.6
-## [40] httr_1.3.1 R6_2.2.2 GenomicAlignments_1.12.2
-## [43] compiler_3.4.2</code></pre>
+## [1] progress_1.1.2 lattice_0.20-35 htmltools_0.3.6
+## [4] yaml_2.1.14 interactiveDisplayBase_1.16.0 blob_1.1.0
+## [7] XML_3.98-1.9 rlang_0.1.4 DBI_0.7
+## [10] BiocParallel_1.12.0 bit64_0.9-7 GenomeInfoDbData_0.99.1
+## [13] stringr_1.2.0 zlibbioc_1.24.0 memoise_1.1.0
+## [16] evaluate_0.10.1 knitr_1.17 biomaRt_2.34.0
+## [19] httpuv_1.3.5 BiocInstaller_1.28.0 curl_3.0
+## [22] Rcpp_0.12.13 xtable_1.8-2 backports_1.1.1
+## [25] mime_0.5 bit_1.1-12 RMySQL_0.10.13
+## [28] digest_0.6.12 stringi_1.1.5 bookdown_0.5
+## [31] shiny_1.0.5 grid_3.4.2 rprojroot_1.2
+## [34] tools_3.4.2 bitops_1.0-6 magrittr_1.5
+## [37] RCurl_1.95-4.8 tibble_1.3.4 RSQLite_2.0
+## [40] pkgconfig_2.0.1 Matrix_1.2-11 prettyunits_1.0.2
+## [43] assertthat_0.2.0 rmarkdown_1.6 httr_1.3.1
+## [46] R6_2.2.2 GenomicAlignments_1.14.0 compiler_3.4.2</code></pre>
+</div>
+
+
+
+
</div>
+<script>
+
+// add bootstrap table styles to pandoc tables
+function bootstrapStylePandocTables() {
+ $('tr.header').parent('thead').parent('table').addClass('table table-condensed');
+}
+$(document).ready(function () {
+ bootstrapStylePandocTables();
+});
+</script>
+
+<script type="text/x-mathjax-config">
+ MathJax.Hub.Config({
+ "HTML-CSS": {
+ styles: {
+ ".MathJax_Display": {
+ "text-align": "center",
+ padding: "0px 150px 0px 65px",
+ margin: "0px 0px 0.5em"
+ },
+ }
+ }
+ });
+</script>
<!-- dynamically load mathjax for compatibility with self-contained -->
<script>
(function () {
diff --git a/inst/doc/AnnotationHub.R b/inst/doc/AnnotationHub.R
index 537392f..d9f599c 100644
--- a/inst/doc/AnnotationHub.R
+++ b/inst/doc/AnnotationHub.R
@@ -1,71 +1,71 @@
-## ----style, echo = FALSE, results = 'asis'-------------------------------
+## ----style, echo = FALSE, results = 'asis'---------------------------------
BiocStyle::markdown()
-## ----library, message=FALSE----------------------------------------------
+## ----library, message=FALSE------------------------------------------------
library(AnnotationHub)
-## ----AnnotationHub-------------------------------------------------------
+## ----AnnotationHub---------------------------------------------------------
ah = AnnotationHub()
-## ----show----------------------------------------------------------------
+## ----show------------------------------------------------------------------
ah
-## ----dataprovider--------------------------------------------------------
+## ----dataprovider----------------------------------------------------------
unique(ah$dataprovider)
-## ----species-------------------------------------------------------------
+## ----species---------------------------------------------------------------
head(unique(ah$species))
-## ----rdataclass----------------------------------------------------------
+## ----rdataclass------------------------------------------------------------
head(unique(ah$rdataclass))
-## ----dm1-----------------------------------------------------------------
+## ----dm1-------------------------------------------------------------------
dm <- query(ah, c("ChainFile", "UCSC", "Drosophila melanogaster"))
dm
-## ----show2---------------------------------------------------------------
+## ----show2-----------------------------------------------------------------
df <- mcols(dm)
-## ----length--------------------------------------------------------------
+## ----length----------------------------------------------------------------
length(ah)
-## ----subset--------------------------------------------------------------
+## ----subset----------------------------------------------------------------
ahs <- query(ah, c('inparanoid8', 'ailuropoda'))
ahs
-## ----display, eval=FALSE-------------------------------------------------
+## ----display, eval=FALSE---------------------------------------------------
# d <- display(ah)
-## ----dm2-----------------------------------------------------------------
+## ----dm2-------------------------------------------------------------------
dm
dm["AH15146"]
-## ----dm3-----------------------------------------------------------------
+## ----dm3-------------------------------------------------------------------
dm[["AH15146"]]
-## ----show-2--------------------------------------------------------------
+## ----show-2----------------------------------------------------------------
ah
-## ----snapshot------------------------------------------------------------
+## ----snapshot--------------------------------------------------------------
snapshotDate(ah)
-## ----possibleDates-------------------------------------------------------
+## ----possibleDates---------------------------------------------------------
pd <- possibleDates(ah)
pd
-## ----setdate, eval=FALSE-------------------------------------------------
+## ----setdate, eval=FALSE---------------------------------------------------
# snapshotDate(ah) <- pd[1]
-## ----clusterOptions1, eval=FALSE-----------------------------------------
+## ----clusterOptions1, eval=FALSE-------------------------------------------
# library(AnnotationHub)
# hub <- AnnotationHub()
# gr <- hub[["AH50773"]] ## downloaded once
# txdb <- makeTxDbFromGRanges(gr) ## build on the fly
-## ----clusterOptions2, eval=FALSE-----------------------------------------
+## ----clusterOptions2, eval=FALSE-------------------------------------------
# library(AnnotationDbi) ## if not already loaded
# txdb <- loadDb("/locationToFile/mytxdb.sqlite")
-## ----sessionInfo---------------------------------------------------------
+## ----sessionInfo-----------------------------------------------------------
sessionInfo()
diff --git a/inst/doc/AnnotationHub.html b/inst/doc/AnnotationHub.html
index 3e00624..5f39440 100644
--- a/inst/doc/AnnotationHub.html
+++ b/inst/doc/AnnotationHub.html
@@ -4,22 +4,28 @@
<head>
-<meta charset="utf-8">
+<meta charset="utf-8" />
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
<meta name="generator" content="pandoc" />
+
<title>AnnotationHub: Access the AnnotationHub Web Service</title>
+<script src="data:application/x-javascript;base64,LyohIGpRdWVyeSB2MS4xMS4zIHwgKGMpIDIwMDUsIDIwMTUgalF1ZXJ5IEZvdW5kYXRpb24sIEluYy4gfCBqcXVlcnkub3JnL2xpY2Vuc2UgKi8KIWZ1bmN0aW9uKGEsYil7Im9iamVjdCI9PXR5cGVvZiBtb2R1bGUmJiJvYmplY3QiPT10eXBlb2YgbW9kdWxlLmV4cG9ydHM/bW9kdWxlLmV4cG9ydHM9YS5kb2N1bWVudD9iKGEsITApOmZ1bmN0aW9uKGEpe2lmKCFhLmRvY3VtZW50KXRocm93IG5ldyBFcnJvcigialF1ZXJ5IHJlcXVpcmVzIGEgd2luZG93IHdpdGggYSBkb2N1bWVudCIpO3JldHVybiBiKGEpfTpiKGEpfSgidW5kZWZpbmVkIiE9dHlwZW9mIHdpbmRvdz93aW5kb3c6dG [...]
+<meta name="viewport" content="width=device-width, initial-scale=1" />
+<link href="data:text/css;charset=utf-8,html%7Bfont%2Dfamily%3Asans%2Dserif%3B%2Dwebkit%2Dtext%2Dsize%2Dadjust%3A100%25%3B%2Dms%2Dtext%2Dsize%2Dadjust%3A100%25%7Dbody%7Bmargin%3A0%7Darticle%2Caside%2Cdetails%2Cfigcaption%2Cfigure%2Cfooter%2Cheader%2Chgroup%2Cmain%2Cmenu%2Cnav%2Csection%2Csummary%7Bdisplay%3Ablock%7Daudio%2Ccanvas%2Cprogress%2Cvideo%7Bdisplay%3Ainline%2Dblock%3Bvertical%2Dalign%3Abaseline%7Daudio%3Anot%28%5Bcontrols%5D%29%7Bdisplay%3Anone%3Bheight%3A0%7D%5Bhidden%5D%2Ctem [...]
+<script src="data:application/x-javascript;base64,LyohCiAqIEJvb3RzdHJhcCB2My4zLjUgKGh0dHA6Ly9nZXRib290c3RyYXAuY29tKQogKiBDb3B5cmlnaHQgMjAxMS0yMDE1IFR3aXR0ZXIsIEluYy4KICogTGljZW5zZWQgdW5kZXIgdGhlIE1JVCBsaWNlbnNlCiAqLwppZigidW5kZWZpbmVkIj09dHlwZW9mIGpRdWVyeSl0aHJvdyBuZXcgRXJyb3IoIkJvb3RzdHJhcCdzIEphdmFTY3JpcHQgcmVxdWlyZXMgalF1ZXJ5Iik7K2Z1bmN0aW9uKGEpeyJ1c2Ugc3RyaWN0Ijt2YXIgYj1hLmZuLmpxdWVyeS5zcGxpdCgiICIpWzBdLnNwbGl0KCIuIik7aWYoYlswXTwyJiZiWzFdPDl8fDE9PWJbMF0mJjk9PWJbMV0mJmJbMl08MSl0aHJvdy [...]
+<script src="data:application/x-javascript;base64,LyoqCiogQHByZXNlcnZlIEhUTUw1IFNoaXYgMy43LjIgfCBAYWZhcmthcyBAamRhbHRvbiBAam9uX25lYWwgQHJlbSB8IE1JVC9HUEwyIExpY2Vuc2VkCiovCi8vIE9ubHkgcnVuIHRoaXMgY29kZSBpbiBJRSA4CmlmICghIXdpbmRvdy5uYXZpZ2F0b3IudXNlckFnZW50Lm1hdGNoKCJNU0lFIDgiKSkgewohZnVuY3Rpb24oYSxiKXtmdW5jdGlvbiBjKGEsYil7dmFyIGM9YS5jcmVhdGVFbGVtZW50KCJwIiksZD1hLmdldEVsZW1lbnRzQnlUYWdOYW1lKCJoZWFkIilbMF18fGEuZG9jdW1lbnRFbGVtZW50O3JldHVybiBjLmlubmVySFRNTD0ieDxzdHlsZT4iK2IrIjwvc3R5bGU+IixkLm [...]
+<script src="data:application/x-javascript;base64,LyohIFJlc3BvbmQuanMgdjEuNC4yOiBtaW4vbWF4LXdpZHRoIG1lZGlhIHF1ZXJ5IHBvbHlmaWxsICogQ29weXJpZ2h0IDIwMTMgU2NvdHQgSmVobAogKiBMaWNlbnNlZCB1bmRlciBodHRwczovL2dpdGh1Yi5jb20vc2NvdHRqZWhsL1Jlc3BvbmQvYmxvYi9tYXN0ZXIvTElDRU5TRS1NSVQKICogICovCgovLyBPbmx5IHJ1biB0aGlzIGNvZGUgaW4gSUUgOAppZiAoISF3aW5kb3cubmF2aWdhdG9yLnVzZXJBZ2VudC5tYXRjaCgiTVNJRSA4IikpIHsKIWZ1bmN0aW9uKGEpeyJ1c2Ugc3RyaWN0IjthLm1hdGNoTWVkaWE9YS5tYXRjaE1lZGlhfHxmdW5jdGlvbihhKXt2YXIgYixjPWEuZG [...]
+<script src="data:application/x-javascript;base64,CgovKioKICogalF1ZXJ5IFBsdWdpbjogU3RpY2t5IFRhYnMKICoKICogQGF1dGhvciBBaWRhbiBMaXN0ZXIgPGFpZGFuQHBocC5uZXQ+CiAqIGFkYXB0ZWQgYnkgUnViZW4gQXJzbGFuIHRvIGFjdGl2YXRlIHBhcmVudCB0YWJzIHRvbwogKiBodHRwOi8vd3d3LmFpZGFubGlzdGVyLmNvbS8yMDE0LzAzL3BlcnNpc3RpbmctdGhlLXRhYi1zdGF0ZS1pbi1ib290c3RyYXAvCiAqLwooZnVuY3Rpb24oJCkgewogICJ1c2Ugc3RyaWN0IjsKICAkLmZuLnJtYXJrZG93blN0aWNreVRhYnMgPSBmdW5jdGlvbigpIHsKICAgIHZhciBjb250ZXh0ID0gdGhpczsKICAgIC8vIFNob3cgdGhlIHRhYi [...]
<link href="data:text/css;charset=utf-8,pre%20%2Eoperator%2C%0Apre%20%2Eparen%20%7B%0Acolor%3A%20rgb%28104%2C%20118%2C%20135%29%0A%7D%0Apre%20%2Eliteral%20%7B%0Acolor%3A%20%23990073%0A%7D%0Apre%20%2Enumber%20%7B%0Acolor%3A%20%23099%3B%0A%7D%0Apre%20%2Ecomment%20%7B%0Acolor%3A%20%23998%3B%0Afont%2Dstyle%3A%20italic%0A%7D%0Apre%20%2Ekeyword%20%7B%0Acolor%3A%20%23900%3B%0Afont%2Dweight%3A%20bold%0A%7D%0Apre%20%2Eidentifier%20%7B%0Acolor%3A%20rgb%280%2C%200%2C%200%29%3B%0A%7D%0Apre%20%2Estri [...]
<script src="data:application/x-javascript;base64,dmFyIGhsanM9bmV3IGZ1bmN0aW9uKCl7ZnVuY3Rpb24gbShwKXtyZXR1cm4gcC5yZXBsYWNlKC8mL2dtLCImYW1wOyIpLnJlcGxhY2UoLzwvZ20sIiZsdDsiKX1mdW5jdGlvbiBmKHIscSxwKXtyZXR1cm4gUmVnRXhwKHEsIm0iKyhyLmNJPyJpIjoiIikrKHA/ImciOiIiKSl9ZnVuY3Rpb24gYihyKXtmb3IodmFyIHA9MDtwPHIuY2hpbGROb2Rlcy5sZW5ndGg7cCsrKXt2YXIgcT1yLmNoaWxkTm9kZXNbcF07aWYocS5ub2RlTmFtZT09IkNPREUiKXtyZXR1cm4gcX1pZighKHEubm9kZVR5cGU9PTMmJnEubm9kZVZhbHVlLm1hdGNoKC9ccysvKSkpe2JyZWFrfX19ZnVuY3Rpb24gaCh0LH [...]
<style type="text/css">code{white-space: pre;}</style>
<style type="text/css">
- pre:not([class]) {
- background-color: white;
- }
+
</style>
<script type="text/javascript">
if (window.hljs && document.readyState && document.readyState === "complete") {
@@ -30,24 +36,84 @@ if (window.hljs && document.readyState && document.readyState === "complete") {
</script>
-<link href="data:text/css;charset=utf-8,body%2C%20td%20%7B%0Afont%2Dfamily%3A%20sans%2Dserif%3B%0Abackground%2Dcolor%3A%20white%3B%0Afont%2Dsize%3A%2013px%3B%0A%7D%0Abody%20%7B%0Amax%2Dwidth%3A%20800px%3B%0Amargin%3A%200%20auto%3B%0Apadding%3A%201em%201em%202em%3B%0Aline%2Dheight%3A%2020px%3B%0A%7D%0A%0Adiv%23TOC%20li%20%7B%0Alist%2Dstyle%3Anone%3B%0Abackground%2Dimage%3Anone%3B%0Abackground%2Drepeat%3Anone%3B%0Abackground%2Dposition%3A0%3B%0A%7D%0A%0Ap%2C%20pre%20%7B%20margin%3A%200em%2 [...]
-<script type="text/javascript">
-document.addEventListener("DOMContentLoaded", function() {
- var links = document.links;
- for (var i = 0, linksLength = links.length; i < linksLength; i++)
- if(links[i].hostname != window.location.hostname)
- links[i].target = '_blank';
-});
-</script>
+<style type="text/css">
+h1 {
+ font-size: 34px;
+}
+h1.title {
+ font-size: 38px;
+}
+h2 {
+ font-size: 30px;
+}
+h3 {
+ font-size: 24px;
+}
+h4 {
+ font-size: 18px;
+}
+h5 {
+ font-size: 16px;
+}
+h6 {
+ font-size: 12px;
+}
+.table th:not([align]) {
+ text-align: left;
+}
+</style>
+
+<link href="data:text/css;charset=utf-8,body%20%7B%0Amargin%3A%200px%20auto%3B%0Amax%2Dwidth%3A%201134px%3B%0Afont%2Dfamily%3A%20sans%2Dserif%3B%0Afont%2Dsize%3A%2010pt%3B%0A%7D%0A%0Adiv%23TOC%20ul%20%7B%0Apadding%3A%200px%200px%200px%2045px%3B%0Alist%2Dstyle%3A%20none%3B%0Abackground%2Dimage%3A%20none%3B%0Abackground%2Drepeat%3A%20none%3B%0Abackground%2Dposition%3A%200%3B%0Afont%2Dsize%3A%2010pt%3B%0Afont%2Dfamily%3A%20Helvetica%2C%20Arial%2C%20sans%2Dserif%3B%0A%7D%0Adiv%23TOC%20%3E%20 [...]
</head>
<body>
+<style type="text/css">
+.main-container {
+ max-width: 828px;
+ margin-left: auto;
+ margin-right: auto;
+}
+
+img {
+ max-width:100%;
+ height: auto;
+}
+.tabbed-pane {
+ padding-top: 12px;
+}
+button.code-folding-btn:focus {
+ outline: none;
+}
+</style>
+
+
+
+<div class="container-fluid main-container">
+
+<!-- tabsets -->
+<script>
+$(document).ready(function () {
+ window.buildTabsets("TOC");
+});
+</script>
+
+<!-- code folding -->
+
+
+
+
+
+
+<div class="fluid-row" id="header">
+
+
+
+<h1 class="title toc-ignore">AnnotationHub: Access the AnnotationHub Web Service</h1>
+<h4 class="date"><em>7 November 2017</em></h4>
-<div id="header">
-<h1 class="title">AnnotationHub: Access the AnnotationHub Web Service</h1>
</div>
<h1>Contents</h1>
@@ -74,7 +140,7 @@ document.addEventListener("DOMContentLoaded", function() {
links[i].target = '_blank';
});
</script>
-<p><strong>Package</strong>: <em><a href="http://bioconductor.org/packages/AnnotationHub">AnnotationHub</a></em><br /> <strong>Authors</strong>: Martin Morgan [cre], Marc Carlson [ctb], Dan Tenenbaum [ctb], Sonali Arora [ctb]<br /> <strong>Modified</strong>: 27 May, 2016<br /> <strong>Compiled</strong>: Tue Oct 17 19:19:56 2017</p>
+<p><strong>Package</strong>: <em><a href="http://bioconductor.org/packages/AnnotationHub">AnnotationHub</a></em><br /> <strong>Authors</strong>: Martin Morgan [cre], Marc Carlson [ctb], Dan Tenenbaum [ctb], Sonali Arora [ctb]<br /> <strong>Modified</strong>: 27 May, 2016<br /> <strong>Compiled</strong>: Tue Nov 7 19:18:46 2017</p>
<p>The <code>AnnotationHub</code> server provides easy <em>R / Bioconductor</em> access to large collections of publicly available whole genome resources, e.g,. ENSEMBL genome fasta or gtf files, UCSC chain resources, ENCODE data tracks at UCSC, etc.</p>
<div id="annotationhub-objects" class="section level1">
<h1><span class="header-section-number">1</span> AnnotationHub objects</h1>
@@ -82,32 +148,32 @@ document.addEventListener("DOMContentLoaded", function() {
<pre class="r"><code>library(AnnotationHub)</code></pre>
<p>The <em><a href="http://bioconductor.org/packages/AnnotationHub">AnnotationHub</a></em> package is straightforward to use. Create an <code>AnnotationHub</code> object</p>
<pre class="r"><code>ah = AnnotationHub()</code></pre>
-<pre><code>## snapshotDate(): 2017-04-25</code></pre>
+<pre><code>## snapshotDate(): 2017-10-27</code></pre>
<p>Now at this point you have already done everything you need in order to start retrieving annotations. For most operations, using the <code>AnnotationHub</code> object should feel a lot like working with a familiar <code>list</code> or <code>data.frame</code>.</p>
<p>Lets take a minute to look at the show method for the hub object ah</p>
<pre class="r"><code>ah</code></pre>
-<pre><code>## AnnotationHub with 40134 records
-## # snapshotDate(): 2017-04-25
-## # $dataprovider: BroadInstitute, Ensembl, UCSC, Haemcode, ftp://ftp.ncb...
-## # $species: Homo sapiens, Mus musculus, Bos taurus, Pan troglodytes, Da...
-## # $rdataclass: GRanges, BigWigFile, FaFile, TwoBitFile, ChainFile, OrgD...
+<pre><code>## AnnotationHub with 42282 records
+## # snapshotDate(): 2017-10-27
+## # $dataprovider: BroadInstitute, Ensembl, UCSC, ftp://ftp.ncbi.nlm.nih.go...
+## # $species: Homo sapiens, Mus musculus, Drosophila melanogaster, Bos taur...
+## # $rdataclass: GRanges, BigWigFile, FaFile, TwoBitFile, Rle, ChainFile, O...
## # additional mcols(): taxonomyid, genome, description,
## # coordinate_1_based, maintainer, rdatadateadded, preparerclass,
## # tags, rdatapath, sourceurl, sourcetype
## # retrieve records with, e.g., 'object[["AH2"]]'
##
-## title
-## AH2 | Ailuropoda_melanoleuca.ailMel1.69.dna.toplevel.fa
-## AH3 | Ailuropoda_melanoleuca.ailMel1.69.dna_rm.toplevel.fa
-## AH4 | Ailuropoda_melanoleuca.ailMel1.69.dna_sm.toplevel.fa
-## AH5 | Ailuropoda_melanoleuca.ailMel1.69.ncrna.fa
-## AH6 | Ailuropoda_melanoleuca.ailMel1.69.pep.all.fa
-## ... ...
-## AH56649 | org.Thermoplasmatales_archaeon_BRNA1.eg.sqlite
-## AH56650 | org.Ignicoccus_hospitalis_KIN4|I.eg.sqlite
-## AH56651 | org.Desulfurococcus_amylolyticus_DSM_16532.eg.sqlite
-## AH56652 | org.Pandoravirus_dulcis.eg.sqlite
-## AH56653 | org.Methanocaldococcus_infernus_ME.eg.sqlite</code></pre>
+## title
+## AH2 | Ailuropoda_melanoleuca.ailMel1.69.dna.toplevel.fa
+## AH3 | Ailuropoda_melanoleuca.ailMel1.69.dna_rm.toplevel.fa
+## AH4 | Ailuropoda_melanoleuca.ailMel1.69.dna_sm.toplevel.fa
+## AH5 | Ailuropoda_melanoleuca.ailMel1.69.ncrna.fa
+## AH6 | Ailuropoda_melanoleuca.ailMel1.69.pep.all.fa
+## ... ...
+## AH58988 | org.Flavobacterium_piscicida.eg.sqlite
+## AH58989 | org.Bacteroides_fragilis_YCH46.eg.sqlite
+## AH58990 | org.Pseudomonas_mendocina_ymp.eg.sqlite
+## AH58991 | org.Salmonella_enterica_subsp._enterica_serovar_Typhimurium_s...
+## AH58992 | org.Acinetobacter_baumannii.eg.sqlite</code></pre>
<p>You can see that it gives you an idea about the different types of data that are present inside the hub. You can see where the data is coming from (dataprovider), as well as what species have samples present (species), what kinds of R data objects could be returned (rdataclass). We can take a closer look at all the kinds of data providers that are available by simply looking at the contents of dataprovider as if it were the column of a data.frame object like this:</p>
<pre class="r"><code>unique(ah$dataprovider)</code></pre>
<pre><code>## [1] "Ensembl"
@@ -122,11 +188,13 @@ document.addEventListener("DOMContentLoaded", function() {
## [10] "BroadInstitute"
## [11] "PRIDE"
## [12] "Gencode"
-## [13] "dbSNP"
-## [14] "CRIBI"
-## [15] "Genoscope"
-## [16] "MISO, VAST-TOOLS, UCSC"
-## [17] "ftp://ftp.ncbi.nlm.nih.gov/gene/DATA/"</code></pre>
+## [13] "CRIBI"
+## [14] "Genoscope"
+## [15] "MISO, VAST-TOOLS, UCSC"
+## [16] "UWashington"
+## [17] "Stanford"
+## [18] "dbSNP"
+## [19] "ftp://ftp.ncbi.nlm.nih.gov/gene/DATA/"</code></pre>
<p>In the same way, you can also see data from different species inside the hub by looking at the contents of species like this:</p>
<pre class="r"><code>head(unique(ah$species))</code></pre>
<pre><code>## [1] "Ailuropoda melanoleuca" "Anolis carolinensis"
@@ -144,7 +212,7 @@ document.addEventListener("DOMContentLoaded", function() {
<pre class="r"><code>dm <- query(ah, c("ChainFile", "UCSC", "Drosophila melanogaster"))
dm</code></pre>
<pre><code>## AnnotationHub with 45 records
-## # snapshotDate(): 2017-04-25
+## # snapshotDate(): 2017-10-27
## # $dataprovider: UCSC
## # $species: Drosophila melanogaster
## # $rdataclass: ChainFile
@@ -170,12 +238,12 @@ dm</code></pre>
<pre class="r"><code>df <- mcols(dm)</code></pre>
<p>By default the show method will only display the first 5 and last 5 rows. There are already thousands of records present in the hub.</p>
<pre class="r"><code>length(ah)</code></pre>
-<pre><code>## [1] 40134</code></pre>
+<pre><code>## [1] 42282</code></pre>
<p>Lets look at another example, where we pull down only Inparanoid8 data from the hub and use subset to return a smaller base object (here we are finding cases where the genome column is set to panda).</p>
<pre class="r"><code>ahs <- query(ah, c('inparanoid8', 'ailuropoda'))
ahs</code></pre>
<pre><code>## AnnotationHub with 1 record
-## # snapshotDate(): 2017-04-25
+## # snapshotDate(): 2017-10-27
## # names(): AH10451
## # $dataprovider: Inparanoid8
## # $species: Ailuropoda melanoleuca
@@ -186,7 +254,7 @@ ahs</code></pre>
## # $taxonomyid: 9646
## # $genome: inparanoid8 genomes
## # $sourcetype: Inparanoid
-## # $sourceurl: http://inparanoid.sbc.su.se/download/current/Orthologs/A....
+## # $sourceurl: http://inparanoid.sbc.su.se/download/current/Orthologs/A.me...
## # $sourcesize: NA
## # $tags: c("Inparanoid", "Gene", "Homology", "Annotation")
## # retrieve record with 'object[["AH10451"]]'</code></pre>
@@ -201,7 +269,7 @@ ahs</code></pre>
<p>Looking back at our chain file example, if we are interested in the file dm1ToDm2.over.chain.gz, we can gets its metadata using</p>
<pre class="r"><code>dm</code></pre>
<pre><code>## AnnotationHub with 45 records
-## # snapshotDate(): 2017-04-25
+## # snapshotDate(): 2017-10-27
## # $dataprovider: UCSC
## # $species: Drosophila melanogaster
## # $rdataclass: ChainFile
@@ -224,7 +292,7 @@ ahs</code></pre>
## AH15146 | dm1ToDm2.over.chain.gz</code></pre>
<pre class="r"><code>dm["AH15146"]</code></pre>
<pre><code>## AnnotationHub with 1 record
-## # snapshotDate(): 2017-04-25
+## # snapshotDate(): 2017-10-27
## # names(): AH15146
## # $dataprovider: UCSC
## # $species: Drosophila melanogaster
@@ -235,7 +303,7 @@ ahs</code></pre>
## # $taxonomyid: 7227
## # $genome: dm1
## # $sourcetype: Chain
-## # $sourceurl: http://hgdownload.cse.ucsc.edu/goldenpath/dm1/liftOver/dm...
+## # $sourceurl: http://hgdownload.cse.ucsc.edu/goldenpath/dm1/liftOver/dm1T...
## # $sourcesize: NA
## # $tags: c("liftOver", "chain", "UCSC", "genome", "homology")
## # retrieve record with 'object[["AH15146"]]'</code></pre>
@@ -251,31 +319,31 @@ ahs</code></pre>
<p>When you create the <code>AnnotationHub</code> object, it will set up the object for you with some default settings. See <code>?AnnotationHub</code> for ways to customize the hub source, the local cache, and other instance-specific options, and <code>?getAnnotationHubOption</code> to get or set package-global options for use across sessions.</p>
<p>If you look at the object you will see some helpful information about it such as where the data is cached and where online the hub server is set to.</p>
<pre class="r"><code>ah</code></pre>
-<pre><code>## AnnotationHub with 40134 records
-## # snapshotDate(): 2017-04-25
-## # $dataprovider: BroadInstitute, Ensembl, UCSC, Haemcode, ftp://ftp.ncb...
-## # $species: Homo sapiens, Mus musculus, Bos taurus, Pan troglodytes, Da...
-## # $rdataclass: GRanges, BigWigFile, FaFile, TwoBitFile, ChainFile, OrgD...
+<pre><code>## AnnotationHub with 42282 records
+## # snapshotDate(): 2017-10-27
+## # $dataprovider: BroadInstitute, Ensembl, UCSC, ftp://ftp.ncbi.nlm.nih.go...
+## # $species: Homo sapiens, Mus musculus, Drosophila melanogaster, Bos taur...
+## # $rdataclass: GRanges, BigWigFile, FaFile, TwoBitFile, Rle, ChainFile, O...
## # additional mcols(): taxonomyid, genome, description,
## # coordinate_1_based, maintainer, rdatadateadded, preparerclass,
## # tags, rdatapath, sourceurl, sourcetype
## # retrieve records with, e.g., 'object[["AH2"]]'
##
-## title
-## AH2 | Ailuropoda_melanoleuca.ailMel1.69.dna.toplevel.fa
-## AH3 | Ailuropoda_melanoleuca.ailMel1.69.dna_rm.toplevel.fa
-## AH4 | Ailuropoda_melanoleuca.ailMel1.69.dna_sm.toplevel.fa
-## AH5 | Ailuropoda_melanoleuca.ailMel1.69.ncrna.fa
-## AH6 | Ailuropoda_melanoleuca.ailMel1.69.pep.all.fa
-## ... ...
-## AH56649 | org.Thermoplasmatales_archaeon_BRNA1.eg.sqlite
-## AH56650 | org.Ignicoccus_hospitalis_KIN4|I.eg.sqlite
-## AH56651 | org.Desulfurococcus_amylolyticus_DSM_16532.eg.sqlite
-## AH56652 | org.Pandoravirus_dulcis.eg.sqlite
-## AH56653 | org.Methanocaldococcus_infernus_ME.eg.sqlite</code></pre>
+## title
+## AH2 | Ailuropoda_melanoleuca.ailMel1.69.dna.toplevel.fa
+## AH3 | Ailuropoda_melanoleuca.ailMel1.69.dna_rm.toplevel.fa
+## AH4 | Ailuropoda_melanoleuca.ailMel1.69.dna_sm.toplevel.fa
+## AH5 | Ailuropoda_melanoleuca.ailMel1.69.ncrna.fa
+## AH6 | Ailuropoda_melanoleuca.ailMel1.69.pep.all.fa
+## ... ...
+## AH58988 | org.Flavobacterium_piscicida.eg.sqlite
+## AH58989 | org.Bacteroides_fragilis_YCH46.eg.sqlite
+## AH58990 | org.Pseudomonas_mendocina_ymp.eg.sqlite
+## AH58991 | org.Salmonella_enterica_subsp._enterica_serovar_Typhimurium_s...
+## AH58992 | org.Acinetobacter_baumannii.eg.sqlite</code></pre>
<p>By default the <code>AnnotationHub</code> object is set to the latest <code>snapshotData</code> and a snapshot version that matches the version of <em>Bioconductor</em> that you are using. You can also learn about these data with the appropriate methods.</p>
<pre class="r"><code>snapshotDate(ah)</code></pre>
-<pre><code>## [1] "2017-04-25"</code></pre>
+<pre><code>## [1] "2017-10-27"</code></pre>
<p>If you are interested in using an older version of a snapshot, you can list previous versions with the <code>possibleDates()</code> like this:</p>
<pre class="r"><code>pd <- possibleDates(ah)
pd</code></pre>
@@ -293,7 +361,10 @@ pd</code></pre>
## [56] "2016-08-15" "2016-10-11" "2016-11-03" "2016-11-08" "2016-11-09"
## [61] "2016-11-13" "2016-11-14" "2016-12-22" "2016-12-28" "2017-01-05"
## [66] "2017-02-07" "2017-04-03" "2017-04-04" "2017-04-05" "2017-04-10"
-## [71] "2017-04-11" "2017-04-13" "2017-04-24" "2017-04-25"</code></pre>
+## [71] "2017-04-11" "2017-04-13" "2017-04-24" "2017-04-25" "2017-05-31"
+## [76] "2017-06-06" "2017-06-07" "2017-06-08" "2017-06-29" "2017-07-11"
+## [81] "2017-08-28" "2017-08-31" "2017-09-07" "2017-10-18" "2017-10-23"
+## [86] "2017-10-24" "2017-10-27" "2017-10-27"</code></pre>
<p>Set the dates like this:</p>
<pre class="r"><code>snapshotDate(ah) <- pd[1]</code></pre>
</div>
@@ -326,8 +397,8 @@ txdb <- loadDb("/locationToFile/mytxdb.sqlite")</code></pre>
## Running under: Ubuntu 16.04.3 LTS
##
## Matrix products: default
-## BLAS: /home/biocbuild/bbs-3.5-bioc/R/lib/libRblas.so
-## LAPACK: /home/biocbuild/bbs-3.5-bioc/R/lib/libRlapack.so
+## BLAS: /home/biocbuild/bbs-3.6-bioc/R/lib/libRblas.so
+## LAPACK: /home/biocbuild/bbs-3.6-bioc/R/lib/libRlapack.so
##
## locale:
## [1] LC_CTYPE=en_US.UTF-8 LC_NUMERIC=C
@@ -342,44 +413,75 @@ txdb <- loadDb("/locationToFile/mytxdb.sqlite")</code></pre>
## [8] methods base
##
## other attached packages:
-## [1] BSgenome.Hsapiens.UCSC.hg19_1.4.0 BSgenome_1.44.2
-## [3] rtracklayer_1.36.6 VariantAnnotation_1.22.3
-## [5] SummarizedExperiment_1.6.5 DelayedArray_0.2.7
-## [7] matrixStats_0.52.2 Rsamtools_1.28.0
-## [9] Biostrings_2.44.2 XVector_0.16.0
-## [11] GenomicFeatures_1.28.5 AnnotationDbi_1.38.2
-## [13] Biobase_2.36.2 GenomicRanges_1.28.6
-## [15] GenomeInfoDb_1.12.3 IRanges_2.10.5
-## [17] S4Vectors_0.14.7 AnnotationHub_2.8.3
-## [19] BiocGenerics_0.22.1 BiocStyle_2.4.1
+## [1] BSgenome.Hsapiens.UCSC.hg19_1.4.0 BSgenome_1.46.0
+## [3] rtracklayer_1.38.0 VariantAnnotation_1.24.1
+## [5] SummarizedExperiment_1.8.0 DelayedArray_0.4.1
+## [7] matrixStats_0.52.2 Rsamtools_1.30.0
+## [9] Biostrings_2.46.0 XVector_0.18.0
+## [11] GenomicFeatures_1.30.0 AnnotationDbi_1.40.0
+## [13] Biobase_2.38.0 GenomicRanges_1.30.0
+## [15] GenomeInfoDb_1.14.0 IRanges_2.12.0
+## [17] S4Vectors_0.16.0 AnnotationHub_2.10.1
+## [19] BiocGenerics_0.24.0 BiocStyle_2.6.0
##
## loaded via a namespace (and not attached):
-## [1] lattice_0.20-35 htmltools_0.3.6
-## [3] yaml_2.1.14 interactiveDisplayBase_1.14.0
-## [5] blob_1.1.0 XML_3.98-1.9
-## [7] rlang_0.1.2 DBI_0.7
-## [9] BiocParallel_1.10.1 bit64_0.9-7
-## [11] GenomeInfoDbData_0.99.0 stringr_1.2.0
-## [13] zlibbioc_1.22.0 memoise_1.1.0
-## [15] evaluate_0.10.1 knitr_1.17
-## [17] biomaRt_2.32.1 httpuv_1.3.5
-## [19] BiocInstaller_1.26.1 curl_3.0
-## [21] Rcpp_0.12.13 xtable_1.8-2
-## [23] backports_1.1.1 mime_0.5
-## [25] bit_1.1-12 digest_0.6.12
-## [27] stringi_1.1.5 shiny_1.0.5
-## [29] rprojroot_1.2 grid_3.4.2
-## [31] tools_3.4.2 bitops_1.0-6
-## [33] magrittr_1.5 RCurl_1.95-4.8
-## [35] tibble_1.3.4 RSQLite_2.0
-## [37] pkgconfig_2.0.1 Matrix_1.2-11
-## [39] rmarkdown_1.6 httr_1.3.1
-## [41] R6_2.2.2 GenomicAlignments_1.12.2
-## [43] compiler_3.4.2</code></pre>
+## [1] progress_1.1.2 lattice_0.20-35
+## [3] htmltools_0.3.6 yaml_2.1.14
+## [5] interactiveDisplayBase_1.16.0 blob_1.1.0
+## [7] XML_3.98-1.9 rlang_0.1.4
+## [9] DBI_0.7 BiocParallel_1.12.0
+## [11] bit64_0.9-7 GenomeInfoDbData_0.99.1
+## [13] stringr_1.2.0 zlibbioc_1.24.0
+## [15] memoise_1.1.0 evaluate_0.10.1
+## [17] knitr_1.17 biomaRt_2.34.0
+## [19] httpuv_1.3.5 BiocInstaller_1.28.0
+## [21] curl_3.0 Rcpp_0.12.13
+## [23] xtable_1.8-2 backports_1.1.1
+## [25] mime_0.5 bit_1.1-12
+## [27] RMySQL_0.10.13 digest_0.6.12
+## [29] stringi_1.1.5 bookdown_0.5
+## [31] shiny_1.0.5 grid_3.4.2
+## [33] rprojroot_1.2 tools_3.4.2
+## [35] bitops_1.0-6 magrittr_1.5
+## [37] RCurl_1.95-4.8 tibble_1.3.4
+## [39] RSQLite_2.0 pkgconfig_2.0.1
+## [41] Matrix_1.2-11 prettyunits_1.0.2
+## [43] assertthat_0.2.0 rmarkdown_1.6
+## [45] httr_1.3.1 R6_2.2.2
+## [47] GenomicAlignments_1.14.0 compiler_3.4.2</code></pre>
+</div>
+
+
+
+
</div>
+<script>
+
+// add bootstrap table styles to pandoc tables
+function bootstrapStylePandocTables() {
+ $('tr.header').parent('thead').parent('table').addClass('table table-condensed');
+}
+$(document).ready(function () {
+ bootstrapStylePandocTables();
+});
+</script>
+
+<script type="text/x-mathjax-config">
+ MathJax.Hub.Config({
+ "HTML-CSS": {
+ styles: {
+ ".MathJax_Display": {
+ "text-align": "center",
+ padding: "0px 150px 0px 65px",
+ margin: "0px 0px 0.5em"
+ },
+ }
+ }
+ });
+</script>
<!-- dynamically load mathjax for compatibility with self-contained -->
<script>
(function () {
diff --git a/inst/doc/CreateAnAnnotationPackage.R b/inst/doc/CreateAnAnnotationPackage.R
new file mode 100644
index 0000000..22f3cee
--- /dev/null
+++ b/inst/doc/CreateAnAnnotationPackage.R
@@ -0,0 +1,25 @@
+## ---- TxDb_Metadata, eval=FALSE--------------------------------------------
+# metadata <- AnnotationHubMetadata(
+# Description="Gene Annotation for Vitis vinifera",
+# Genome="IGGP12Xv0",
+# Species="Vitis vinifera",
+# SourceUrl="http://genomes.cribi.unipd.it/DATA/V2/V2.1/V2.1.gff3",
+# SourceLastModifiedDate=as.POSIXct("2014-04-17"),
+# SourceVersion="2.1",
+# RDataPath="community/tflutre/",
+# TaxonomyId=29760L,
+# Title="Vvinifera_CRIBI_IGGP12Xv0_V2.1.gff3.Rdata",
+# BiocVersion=package_version("3.3"),
+# Coordinate_1_based=TRUE,
+# DataProvider="CRIBI",
+# Maintainer="Timothée Flutre <timothee.flutre at supagro.inra.fr",
+# RDataClass="GRanges",
+# DispatchClass="GRanges",
+# SourceType="GFF",
+# RDataDateAdded=as.POSIXct(Sys.time()),
+# Recipe=NA_character_,
+# PreparerClass="None",
+# Tags=c("GFF", "CRIBI", "Gene", "Transcript", "Annotation"),
+# Notes="chrUn renamed to chrUkn"
+# )
+
diff --git a/inst/doc/CreateAnAnnotationPackage.Rmd b/inst/doc/CreateAnAnnotationPackage.Rmd
new file mode 100644
index 0000000..51fa8da
--- /dev/null
+++ b/inst/doc/CreateAnAnnotationPackage.Rmd
@@ -0,0 +1,354 @@
+---
+title: "Creating An AnnotationHub Package"
+author: "Valerie Obenchain and Lori Shepherd"
+date: "Modified: October 2016. Compiled: `r format(Sys.Date(), '%d %b %Y')`"
+output:
+ BiocStyle::html_document:
+ toc: true
+vignette: >
+ % \VignetteIndexEntry{AnnotationHub: Creating An AnnotationHub Package}
+ % \VignetteEngine{knitr::rmarkdown}
+ % \VignetteEncoding{UTF-8}
+---
+
+
+# Overview
+
+The `AnnotationHubData` package provides tools to acquire, annotate, convert
+and store data for use in Bioconductor's `AnnotationHub`. BED files from the
+Encode project, gtf files from Ensembl, or annotation tracks from UCSC, are
+examples of data that can be downloaded, described with metadata, transformed
+to standard `Bioconductor` data types, and stored so that they may be
+conveniently served up on demand to users via the AnnotationHub client. While
+data are often manipulated into a more R-friendly form, the data themselves
+retain their raw content and are not filtered or curated like those in
+[ExperimentHub](http://bioconductor.org/packages/ExperimentHub/).
+Each resource has associated metadata that can be searched through the
+`AnnotationHub` client interface.
+
+# New resources
+
+## Family of resources
+
+Multiple, related resources are added to `AnnotationHub` by creating a software
+package similar to the existing annotation packages. The package itself does
+not contain data but serves as a light weight wrapper around scripts that
+generate metadata for the resources added to `AnnotationHub`.
+
+At a minimum the package should contain a man page describing the resources.
+Vignettes and additional `R` code for manipulating the objects are optional.
+
+Creating the package involves the following steps:
+
+1. Notify `Bioconductor` team member:
+ Man page and vignette examples in the software package will not work until
+ the data are available in `AnnotationHub`. Adding the data to AWS S3 and the
+ metadata to the production database involves assistance from a `Bioconductor`
+ team member. Please read the section "Uploading Data to S3".
+
+2. Building the software package:
+ Below is an outline of package organization. The files listed are required
+ unless otherwise stated.
+
+* inst/extdata/
+
+ - metadata.csv:
+ This file contains the metadata in the format of one row per resource
+ to be added to the `AnnotationHub` database. The file should be generated
+ from the code in inst/scripts/make-metadata.R where the final data are
+ written out with write.csv(..., row.names=FALSE). The required column
+ names and data types are specified in
+ `AnnotationHub::readMetadataFromCsv()`. See ?`readMetadataFromCsv` for
+ details.
+
+ If necessary, metadata can be broken up into multiple csv files instead
+ having of all records in a single "metadata.csv".
+
+* inst/scripts/
+
+ - make-data.R:
+ A script describing the steps involved in making the data object(s). This
+ includes where the original data were downloaded from, pre-processing, and
+ how the final R object was made. Include a description of any steps
+ performed outside of `R` with third party software. Output of the script
+ should be files on disk ready to be pushed to S3. If data are to be hosted
+ on a personal web site instead of S3, this file should explain any
+ manipulation of the data prior to hosting on the web site. For data hosted
+ on a public web site with no prior manipultaion this file is not needed.
+
+ - make-metadata.R:
+ A script to make the metadata.csv file located in inst/extdata of the
+ package. See ?`readMetadataFromCsv` for a description of the
+ metadata.csv file, expected fields and data types. The
+ `readMetadataFromCsv()` function can be used to validate the metadata.csv
+ file before submitting the package.
+
+* vignettes/
+
+ OPTIONAL vignette(s) describing analysis workflows.
+
+* R/
+
+ OPTIONAL functions to enhance data exploration.
+
+* man/
+
+ - package man page:
+ OPTIONAL. The package man page serves as a landing point and should briefly
+ describe all resources associated with the package. There should be an
+ \alias entry for each resource title either on the package man page or
+ individual man pages.
+
+ - resource man pages:
+ OPTIONAL. Man page(s) should describe the resource (raw data source,
+ processing, QC steps) and demonstrate how the data can be loaded through
+ the `AnnotationHub` interface. For example, replace "SEARCHTERM*" below
+ with one or more search terms that uniquely identify resources in your
+ package.
+
+ ```
+ library(AnnotationHub)
+ hub <- AnnotationHub()
+ myfiles <- query(hub, "SEARCHTERM1", "SEARCHTERM2")
+ myfiles[[1]] ## load the first resource in the list
+ ```
+
+* DESCRIPTION / NAMESPACE
+The scripts used to generate the metadata will likely use functions from
+AnnotationHub or AnnotationHubData which should be listed in Depends/Imports as
+necessary.
+
+
+3. Data objects:
+Data are not formally part of the software package and are stored
+separately in AWS S3 buckets. The author should follow instructions in the
+section "Uploading Data to S3"
+
+4. Confirm valid metadata:
+Confirm the data in inst/exdata/metadata.csv are valid by running
+AnnotationHubData:::makeAnnotationHubMetadata() on your package. Please
+address and warnings or errors.
+
+5. Package review:
+Submit the package to the
+[tracker](https://github.com/Bioconductor/Contributions) for review. The
+primary purpose of the package review is to validate the metadata in the csv
+file(s). It is ok if the package fails R CMD build and check because the
+data and metadata are not yet in place. Once the metadata.csv is approved,
+records are added to the production database. At that point the package man
+pages and vignette can be finalized and the package should pass R CMD build
+and check.
+
+## Individual resources
+
+Individual objects of a standard class can be added to the hub by providing
+only the data and metadata files or by creating a package as described in the
+`Family of Resources` section.
+
+OrgDb, TxDb and BSgenome objects are well defined `Bioconductor` classes and
+methods to download and process these objects already exist in `AnnotationHub`.
+When adding only one or two objects the overhead of creating a package may be
+unnecessary. The goal of the package is to provide structure for metadata
+generation and makes sense when there are plans to update versions or add new
+organisms in the future.
+
+Make sure the OrgDb, TxDb or BSgenome object you want to add does not already
+exist in the
+[Biocondcutor annotation repository](http://www.bioconductor.org/packages/release/BiocViews.html#___AnnotationData)
+
+Providing just data and metadata files involves the following steps:
+
+1. Notify `Bioconductor` team member:
+ Adding the data to AWS S3 and the metadata to the production database
+ involves assistance from a `Bioconductor` team member. Please read the
+ section "Uploading Data to S3".
+
+2. Prepare the data:
+ In the case of an OrgDb object, only the sqlite file is stored in S3.
+ See makeOrgPackageFromNCBI() and makeOrgPackage() in the `AnnotationForge`
+ package for help creating the sqlite file. BSgenome objects should be made
+ according to the steps outline in the
+ [BSgenome
+ vignette](http://www.bioconductor.org/packages/3.4/bioc/vignettes/BSgenome/inst/doc/BSgenomeForge.pdf). TxDb objects will be made on-the-fly from a
+ GRanges with GenomicFeatures::makeTxDbFromGRanges() when the resource is
+ downloaded from `AnnotationHub`. Data should be provided as a GRanges
+ object. See GenomicRanges::makeGRangesFromDataFrame() or
+ rtracklayer::import() for help creating the GRanges.
+
+3. Generate metadata:
+ Prepare a .R file that generates metadata for the resource(s) by calling
+ the `AnnotationHubData::AnnotationHubMetadata()` constructor. Argument
+ details are found on the ?`AnnotationHubMetadata` man page.
+
+ As an example, this piece of code generates the metadata for
+ the Vitis vinifera TxDb Timothée Flutre contributed to `AnnotationHub`:
+
+```{r, TxDb_Metadata, eval=FALSE}
+metadata <- AnnotationHubMetadata(
+ Description="Gene Annotation for Vitis vinifera",
+ Genome="IGGP12Xv0",
+ Species="Vitis vinifera",
+ SourceUrl="http://genomes.cribi.unipd.it/DATA/V2/V2.1/V2.1.gff3",
+ SourceLastModifiedDate=as.POSIXct("2014-04-17"),
+ SourceVersion="2.1",
+ RDataPath="community/tflutre/",
+ TaxonomyId=29760L,
+ Title="Vvinifera_CRIBI_IGGP12Xv0_V2.1.gff3.Rdata",
+ BiocVersion=package_version("3.3"),
+ Coordinate_1_based=TRUE,
+ DataProvider="CRIBI",
+ Maintainer="Timothée Flutre <timothee.flutre at supagro.inra.fr",
+ RDataClass="GRanges",
+ DispatchClass="GRanges",
+ SourceType="GFF",
+ RDataDateAdded=as.POSIXct(Sys.time()),
+ Recipe=NA_character_,
+ PreparerClass="None",
+ Tags=c("GFF", "CRIBI", "Gene", "Transcript", "Annotation"),
+ Notes="chrUn renamed to chrUkn"
+)
+```
+
+4. Add data to S3 and metadata to the database:
+ This last step is done by the `Biocondcutor` team member.
+
+# Additional resources
+
+Metadata for new versions of the data can be added to the same package as they
+become available.
+
+* The titles for the new versions should be unique and not match the title of
+ any resource currently in AnnotationHub. Good practice would be to
+ include the version and / or genome build in the title. If the title is
+ not unique, the `AnnotationHub` object will list multiple files with the
+ same title. The user will need to use 'rdatadateadded' to determine which
+ is the most current.
+
+* Make data available: see section on "Uploading Data to S3"
+
+* Update make-metadata.R with the new metadata information
+
+* Generate a new metadata.csv file. The package should contain
+ metadata for all versions of the data in AnnotationHub so the old file should
+ remain. When adding a new version it might be helpful to write a new csv
+ file named by version, e.g., metadata_v84.csv, metadata_85.csv etc.
+
+* Bump package version and commit to git
+
+* Notify Lori.Shepherd at Roswellpark.org that an update is ready and
+ a team member will add the new metadata to the production database;
+ new resources will not be visible in AnnotationHub until
+ the metadata are added to the database.
+
+Contact Lori.Shepherd at roswellpark.org or maintainer at bioconductor.org with any
+questions.
+
+# Bug fixes
+
+A bug fix may involve a change to the metadata, data resource or both.
+
+## Update the resource
+
+* The replacement resource must have the same name as the original and
+ be at the same location (path).
+
+* Notify Lori.Shepherd at roswellpark.org that you want to replace the data
+ and make the files available: see section "Uploading Data to S3".
+
+## Update the metadata
+
+New metadata records can be added for new resources but modifying existing
+records is discouraged. Record modification will only be done in the case of
+bug fixes.
+
+* Notify Lori.Shepherd at roswellpark.org that you want to change the metadata
+
+* Update make-metadata.R and regenerate the metadata.csv file
+
+* Bump the package version and commit to git
+
+# Remove resources
+
+When a resource is removed from `AnnotationHub` two things happen:
+the 'rdatadateremoved' field is populated with a date and the 'status'
+field is populated with a reason why the resource is no longer available. Once
+these changes are made, the `AnnotationHub()` constructor will not list the
+resource among the available ids. An attempt to extract the resource with
+'[[' and the AH id will return an error along with the status message.
+
+In general, resources are only removed when they are no longer available
+(e.g., moved from web location, no longer provided etc.).
+
+To remove a resource from `AnnotationHub` contact Lori.Shepherd at roswellpark.org
+or maintainer at bioconductor.org.
+
+# Versioning
+
+Versioning of resources is handled by the maintainer. If you plan to provide
+incremental updates to a file for the same organism / genome build, we
+recommend including a version in the title of the resource so it is easy
+to distinguish which is most current.
+
+If you do not include a version, or make the title unique in some way,
+multiple files with the same title will be listed in the `AnnotationHub`
+object. The user will can use the 'rdatadateadded' metadata field
+to determine which file is the most current.
+
+# Visibility
+
+Several metadata fields control which resources are visible when
+a user invokes AnnotationHub(). Records are filtered based on these criteria:
+
+- 'snapshotdate' >= the date of the Bioconductor release being used
+- 'rdatadateadded' >= today's date
+- 'rdatadateremoved' is NULL / NA
+- 'biocVersion' is <= to the Bioconductor version being used
+
+Once a record is added to AnnotationHub it is visable from that point forward
+until stamped with 'rdatadateremoved'. For example, a record added on
+May 1, 2017 with 'biocVersion' 3.6 will be visible in all snapshots >=
+May1, 2017 and in all Bioconductor versions >= 3.6.
+
+# Uploading Data to S3
+
+Instead of providing the data files via dropbox, ftp, etc. we will grant
+temporary access to an S3 bucket where you can upload your data. Please
+email Lori.Shepherd at roswellpark.org for access.
+
+You will be given access to the 'AnnotationContributor' user. Ensure that the
+`AWS CLI` is installed on your machine. See instructions for installing `AWS
+CLI` [here](https://aws.amazon.com/cli/). Once you have requested access you
+will be emailed a set of keys. There are two options to set the profile up for
+AnnotationContributor
+
+1. Update your `.aws/config` file to include the following updating the keys
+accordingly:
+
+```
+[profile AnnotationContributor]
+output = text
+region = us-east-1
+aws_access_key_id = ****
+aws_secret_access_key = ****
+```
+2. If you can't find the `.aws/config` file, Run the following command entering
+appropriate information from above
+
+```
+aws configure --profile AnnotationContributor
+```
+
+After the configuration is set you should be able to upload resources using
+
+```
+aws --profile AnnotationContributor s3 cp test_file.txt s3://annotation-contributor/test_file.txt --acl public-read
+
+```
+
+Please upload the data with the appropriate directory structure, including
+subdirectories as necessary (i.e. top directory must be software package name,
+then if applicable, subdirectories of versions, ...)
+
+Once the upload is complete, email Lori.Shepherd at roswellpark.org to continue the
+process
+
diff --git a/inst/doc/CreateAnAnnotationPackage.html b/inst/doc/CreateAnAnnotationPackage.html
new file mode 100644
index 0000000..6190d5c
--- /dev/null
+++ b/inst/doc/CreateAnAnnotationPackage.html
@@ -0,0 +1,364 @@
+<!DOCTYPE html>
+
+<html xmlns="http://www.w3.org/1999/xhtml">
+
+<head>
+
+<meta charset="utf-8" />
+<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+<meta name="generator" content="pandoc" />
+
+
+<meta name="author" content="Valerie Obenchain and Lori Shepherd" />
+
+
+<title>Creating An AnnotationHub Package</title>
+
+<script src="data:application/x-javascript;base64,LyohIGpRdWVyeSB2MS4xMS4zIHwgKGMpIDIwMDUsIDIwMTUgalF1ZXJ5IEZvdW5kYXRpb24sIEluYy4gfCBqcXVlcnkub3JnL2xpY2Vuc2UgKi8KIWZ1bmN0aW9uKGEsYil7Im9iamVjdCI9PXR5cGVvZiBtb2R1bGUmJiJvYmplY3QiPT10eXBlb2YgbW9kdWxlLmV4cG9ydHM/bW9kdWxlLmV4cG9ydHM9YS5kb2N1bWVudD9iKGEsITApOmZ1bmN0aW9uKGEpe2lmKCFhLmRvY3VtZW50KXRocm93IG5ldyBFcnJvcigialF1ZXJ5IHJlcXVpcmVzIGEgd2luZG93IHdpdGggYSBkb2N1bWVudCIpO3JldHVybiBiKGEpfTpiKGEpfSgidW5kZWZpbmVkIiE9dHlwZW9mIHdpbmRvdz93aW5kb3c6dG [...]
+<meta name="viewport" content="width=device-width, initial-scale=1" />
+<link href="data:text/css;charset=utf-8,html%7Bfont%2Dfamily%3Asans%2Dserif%3B%2Dwebkit%2Dtext%2Dsize%2Dadjust%3A100%25%3B%2Dms%2Dtext%2Dsize%2Dadjust%3A100%25%7Dbody%7Bmargin%3A0%7Darticle%2Caside%2Cdetails%2Cfigcaption%2Cfigure%2Cfooter%2Cheader%2Chgroup%2Cmain%2Cmenu%2Cnav%2Csection%2Csummary%7Bdisplay%3Ablock%7Daudio%2Ccanvas%2Cprogress%2Cvideo%7Bdisplay%3Ainline%2Dblock%3Bvertical%2Dalign%3Abaseline%7Daudio%3Anot%28%5Bcontrols%5D%29%7Bdisplay%3Anone%3Bheight%3A0%7D%5Bhidden%5D%2Ctem [...]
+<script src="data:application/x-javascript;base64,LyohCiAqIEJvb3RzdHJhcCB2My4zLjUgKGh0dHA6Ly9nZXRib290c3RyYXAuY29tKQogKiBDb3B5cmlnaHQgMjAxMS0yMDE1IFR3aXR0ZXIsIEluYy4KICogTGljZW5zZWQgdW5kZXIgdGhlIE1JVCBsaWNlbnNlCiAqLwppZigidW5kZWZpbmVkIj09dHlwZW9mIGpRdWVyeSl0aHJvdyBuZXcgRXJyb3IoIkJvb3RzdHJhcCdzIEphdmFTY3JpcHQgcmVxdWlyZXMgalF1ZXJ5Iik7K2Z1bmN0aW9uKGEpeyJ1c2Ugc3RyaWN0Ijt2YXIgYj1hLmZuLmpxdWVyeS5zcGxpdCgiICIpWzBdLnNwbGl0KCIuIik7aWYoYlswXTwyJiZiWzFdPDl8fDE9PWJbMF0mJjk9PWJbMV0mJmJbMl08MSl0aHJvdy [...]
+<script src="data:application/x-javascript;base64,LyoqCiogQHByZXNlcnZlIEhUTUw1IFNoaXYgMy43LjIgfCBAYWZhcmthcyBAamRhbHRvbiBAam9uX25lYWwgQHJlbSB8IE1JVC9HUEwyIExpY2Vuc2VkCiovCi8vIE9ubHkgcnVuIHRoaXMgY29kZSBpbiBJRSA4CmlmICghIXdpbmRvdy5uYXZpZ2F0b3IudXNlckFnZW50Lm1hdGNoKCJNU0lFIDgiKSkgewohZnVuY3Rpb24oYSxiKXtmdW5jdGlvbiBjKGEsYil7dmFyIGM9YS5jcmVhdGVFbGVtZW50KCJwIiksZD1hLmdldEVsZW1lbnRzQnlUYWdOYW1lKCJoZWFkIilbMF18fGEuZG9jdW1lbnRFbGVtZW50O3JldHVybiBjLmlubmVySFRNTD0ieDxzdHlsZT4iK2IrIjwvc3R5bGU+IixkLm [...]
+<script src="data:application/x-javascript;base64,LyohIFJlc3BvbmQuanMgdjEuNC4yOiBtaW4vbWF4LXdpZHRoIG1lZGlhIHF1ZXJ5IHBvbHlmaWxsICogQ29weXJpZ2h0IDIwMTMgU2NvdHQgSmVobAogKiBMaWNlbnNlZCB1bmRlciBodHRwczovL2dpdGh1Yi5jb20vc2NvdHRqZWhsL1Jlc3BvbmQvYmxvYi9tYXN0ZXIvTElDRU5TRS1NSVQKICogICovCgovLyBPbmx5IHJ1biB0aGlzIGNvZGUgaW4gSUUgOAppZiAoISF3aW5kb3cubmF2aWdhdG9yLnVzZXJBZ2VudC5tYXRjaCgiTVNJRSA4IikpIHsKIWZ1bmN0aW9uKGEpeyJ1c2Ugc3RyaWN0IjthLm1hdGNoTWVkaWE9YS5tYXRjaE1lZGlhfHxmdW5jdGlvbihhKXt2YXIgYixjPWEuZG [...]
+<script src="data:application/x-javascript;base64,CgovKioKICogalF1ZXJ5IFBsdWdpbjogU3RpY2t5IFRhYnMKICoKICogQGF1dGhvciBBaWRhbiBMaXN0ZXIgPGFpZGFuQHBocC5uZXQ+CiAqIGFkYXB0ZWQgYnkgUnViZW4gQXJzbGFuIHRvIGFjdGl2YXRlIHBhcmVudCB0YWJzIHRvbwogKiBodHRwOi8vd3d3LmFpZGFubGlzdGVyLmNvbS8yMDE0LzAzL3BlcnNpc3RpbmctdGhlLXRhYi1zdGF0ZS1pbi1ib290c3RyYXAvCiAqLwooZnVuY3Rpb24oJCkgewogICJ1c2Ugc3RyaWN0IjsKICAkLmZuLnJtYXJrZG93blN0aWNreVRhYnMgPSBmdW5jdGlvbigpIHsKICAgIHZhciBjb250ZXh0ID0gdGhpczsKICAgIC8vIFNob3cgdGhlIHRhYi [...]
+<link href="data:text/css;charset=utf-8,pre%20%2Eoperator%2C%0Apre%20%2Eparen%20%7B%0Acolor%3A%20rgb%28104%2C%20118%2C%20135%29%0A%7D%0Apre%20%2Eliteral%20%7B%0Acolor%3A%20%23990073%0A%7D%0Apre%20%2Enumber%20%7B%0Acolor%3A%20%23099%3B%0A%7D%0Apre%20%2Ecomment%20%7B%0Acolor%3A%20%23998%3B%0Afont%2Dstyle%3A%20italic%0A%7D%0Apre%20%2Ekeyword%20%7B%0Acolor%3A%20%23900%3B%0Afont%2Dweight%3A%20bold%0A%7D%0Apre%20%2Eidentifier%20%7B%0Acolor%3A%20rgb%280%2C%200%2C%200%29%3B%0A%7D%0Apre%20%2Estri [...]
+<script src="data:application/x-javascript;base64,dmFyIGhsanM9bmV3IGZ1bmN0aW9uKCl7ZnVuY3Rpb24gbShwKXtyZXR1cm4gcC5yZXBsYWNlKC8mL2dtLCImYW1wOyIpLnJlcGxhY2UoLzwvZ20sIiZsdDsiKX1mdW5jdGlvbiBmKHIscSxwKXtyZXR1cm4gUmVnRXhwKHEsIm0iKyhyLmNJPyJpIjoiIikrKHA/ImciOiIiKSl9ZnVuY3Rpb24gYihyKXtmb3IodmFyIHA9MDtwPHIuY2hpbGROb2Rlcy5sZW5ndGg7cCsrKXt2YXIgcT1yLmNoaWxkTm9kZXNbcF07aWYocS5ub2RlTmFtZT09IkNPREUiKXtyZXR1cm4gcX1pZighKHEubm9kZVR5cGU9PTMmJnEubm9kZVZhbHVlLm1hdGNoKC9ccysvKSkpe2JyZWFrfX19ZnVuY3Rpb24gaCh0LH [...]
+
+<style type="text/css">code{white-space: pre;}</style>
+<style type="text/css">
+
+</style>
+<script type="text/javascript">
+if (window.hljs && document.readyState && document.readyState === "complete") {
+ window.setTimeout(function() {
+ hljs.initHighlighting();
+ }, 0);
+}
+</script>
+
+
+
+<style type="text/css">
+h1 {
+ font-size: 34px;
+}
+h1.title {
+ font-size: 38px;
+}
+h2 {
+ font-size: 30px;
+}
+h3 {
+ font-size: 24px;
+}
+h4 {
+ font-size: 18px;
+}
+h5 {
+ font-size: 16px;
+}
+h6 {
+ font-size: 12px;
+}
+.table th:not([align]) {
+ text-align: left;
+}
+</style>
+
+<link href="data:text/css;charset=utf-8,body%20%7B%0Amargin%3A%200px%20auto%3B%0Amax%2Dwidth%3A%201134px%3B%0Afont%2Dfamily%3A%20sans%2Dserif%3B%0Afont%2Dsize%3A%2010pt%3B%0A%7D%0A%0Adiv%23TOC%20ul%20%7B%0Apadding%3A%200px%200px%200px%2045px%3B%0Alist%2Dstyle%3A%20none%3B%0Abackground%2Dimage%3A%20none%3B%0Abackground%2Drepeat%3A%20none%3B%0Abackground%2Dposition%3A%200%3B%0Afont%2Dsize%3A%2010pt%3B%0Afont%2Dfamily%3A%20Helvetica%2C%20Arial%2C%20sans%2Dserif%3B%0A%7D%0Adiv%23TOC%20%3E%20 [...]
+
+</head>
+
+<body>
+
+<style type="text/css">
+.main-container {
+ max-width: 828px;
+ margin-left: auto;
+ margin-right: auto;
+}
+
+img {
+ max-width:100%;
+ height: auto;
+}
+.tabbed-pane {
+ padding-top: 12px;
+}
+button.code-folding-btn:focus {
+ outline: none;
+}
+</style>
+
+
+
+<div class="container-fluid main-container">
+
+<!-- tabsets -->
+<script>
+$(document).ready(function () {
+ window.buildTabsets("TOC");
+});
+</script>
+
+<!-- code folding -->
+
+
+
+
+
+
+<div class="fluid-row" id="header">
+
+
+
+<h1 class="title toc-ignore">Creating An AnnotationHub Package</h1>
+<p class="author-name">Valerie Obenchain and Lori Shepherd</p>
+<h4 class="date"><em>Modified: October 2016. Compiled: 07 Nov 2017</em></h4>
+
+</div>
+
+<h1>Contents</h1>
+<div id="TOC">
+<ul>
+<li><a href="#overview"><span class="toc-section-number">1</span> Overview</a></li>
+<li><a href="#new-resources"><span class="toc-section-number">2</span> New resources</a><ul>
+<li><a href="#family-of-resources"><span class="toc-section-number">2.1</span> Family of resources</a></li>
+<li><a href="#individual-resources"><span class="toc-section-number">2.2</span> Individual resources</a></li>
+</ul></li>
+<li><a href="#additional-resources"><span class="toc-section-number">3</span> Additional resources</a></li>
+<li><a href="#bug-fixes"><span class="toc-section-number">4</span> Bug fixes</a><ul>
+<li><a href="#update-the-resource"><span class="toc-section-number">4.1</span> Update the resource</a></li>
+<li><a href="#update-the-metadata"><span class="toc-section-number">4.2</span> Update the metadata</a></li>
+</ul></li>
+<li><a href="#remove-resources"><span class="toc-section-number">5</span> Remove resources</a></li>
+<li><a href="#versioning"><span class="toc-section-number">6</span> Versioning</a></li>
+<li><a href="#visibility"><span class="toc-section-number">7</span> Visibility</a></li>
+<li><a href="#uploading-data-to-s3"><span class="toc-section-number">8</span> Uploading Data to S3</a></li>
+</ul>
+</div>
+
+<div id="overview" class="section level1">
+<h1><span class="header-section-number">1</span> Overview</h1>
+<p>The <code>AnnotationHubData</code> package provides tools to acquire, annotate, convert and store data for use in Bioconductor’s <code>AnnotationHub</code>. BED files from the Encode project, gtf files from Ensembl, or annotation tracks from UCSC, are examples of data that can be downloaded, described with metadata, transformed to standard <code>Bioconductor</code> data types, and stored so that they may be conveniently served up on demand to users via the AnnotationHub client. While [...]
+Each resource has associated metadata that can be searched through the <code>AnnotationHub</code> client interface.</p>
+</div>
+<div id="new-resources" class="section level1">
+<h1><span class="header-section-number">2</span> New resources</h1>
+<div id="family-of-resources" class="section level2">
+<h2><span class="header-section-number">2.1</span> Family of resources</h2>
+<p>Multiple, related resources are added to <code>AnnotationHub</code> by creating a software package similar to the existing annotation packages. The package itself does not contain data but serves as a light weight wrapper around scripts that generate metadata for the resources added to <code>AnnotationHub</code>.</p>
+<p>At a minimum the package should contain a man page describing the resources. Vignettes and additional <code>R</code> code for manipulating the objects are optional.</p>
+<p>Creating the package involves the following steps:</p>
+<ol style="list-style-type: decimal">
+<li><p>Notify <code>Bioconductor</code> team member:<br />
+Man page and vignette examples in the software package will not work until the data are available in <code>AnnotationHub</code>. Adding the data to AWS S3 and the metadata to the production database involves assistance from a <code>Bioconductor</code> team member. Please read the section “Uploading Data to S3”.</p></li>
+<li><p>Building the software package:<br />
+Below is an outline of package organization. The files listed are required unless otherwise stated.</p></li>
+</ol>
+<ul>
+<li><p>inst/extdata/</p>
+<ul>
+<li>metadata.csv: This file contains the metadata in the format of one row per resource to be added to the <code>AnnotationHub</code> database. The file should be generated from the code in inst/scripts/make-metadata.R where the final data are written out with write.csv(…, row.names=FALSE). The required column names and data types are specified in <code>AnnotationHub::readMetadataFromCsv()</code>. See ?<code>readMetadataFromCsv</code> for details.</li>
+</ul>
+<p>If necessary, metadata can be broken up into multiple csv files instead having of all records in a single “metadata.csv”.</p></li>
+<li><p>inst/scripts/</p>
+<ul>
+<li><p>make-data.R:<br />
+A script describing the steps involved in making the data object(s). This includes where the original data were downloaded from, pre-processing, and how the final R object was made. Include a description of any steps performed outside of <code>R</code> with third party software. Output of the script should be files on disk ready to be pushed to S3. If data are to be hosted on a personal web site instead of S3, this file should explain any manipulation of the data prior to hosting on the [...]
+<li><p>make-metadata.R:<br />
+A script to make the metadata.csv file located in inst/extdata of the package. See ?<code>readMetadataFromCsv</code> for a description of the metadata.csv file, expected fields and data types. The <code>readMetadataFromCsv()</code> function can be used to validate the metadata.csv file before submitting the package.</p></li>
+</ul></li>
+<li><p>vignettes/</p>
+<p>OPTIONAL vignette(s) describing analysis workflows.</p></li>
+<li><p>R/</p>
+<p>OPTIONAL functions to enhance data exploration.</p></li>
+<li><p>man/</p>
+<ul>
+<li><p>package man page:<br />
+OPTIONAL. The package man page serves as a landing point and should briefly describe all resources associated with the package. There should be an entry for each resource title either on the package man page or individual man pages.</p></li>
+<li><p>resource man pages:<br />
+OPTIONAL. Man page(s) should describe the resource (raw data source, processing, QC steps) and demonstrate how the data can be loaded through the <code>AnnotationHub</code> interface. For example, replace “SEARCHTERM*" below with one or more search terms that uniquely identify resources in your package.</p></li>
+</ul>
+<pre><code>library(AnnotationHub)
+hub <- AnnotationHub()
+myfiles <- query(hub, "SEARCHTERM1", "SEARCHTERM2")
+myfiles[[1]] ## load the first resource in the list</code></pre></li>
+<li><p>DESCRIPTION / NAMESPACE<br />
+The scripts used to generate the metadata will likely use functions from AnnotationHub or AnnotationHubData which should be listed in Depends/Imports as necessary.</p></li>
+</ul>
+<ol start="3" style="list-style-type: decimal">
+<li><p>Data objects:<br />
+Data are not formally part of the software package and are stored separately in AWS S3 buckets. The author should follow instructions in the section “Uploading Data to S3”</p></li>
+<li><p>Confirm valid metadata:<br />
+Confirm the data in inst/exdata/metadata.csv are valid by running AnnotationHubData:::makeAnnotationHubMetadata() on your package. Please address and warnings or errors.</p></li>
+<li><p>Package review:<br />
+Submit the package to the <a href="https://github.com/Bioconductor/Contributions">tracker</a> for review. The primary purpose of the package review is to validate the metadata in the csv file(s). It is ok if the package fails R CMD build and check because the data and metadata are not yet in place. Once the metadata.csv is approved, records are added to the production database. At that point the package man pages and vignette can be finalized and the package should pass R CMD build and c [...]
+</ol>
+</div>
+<div id="individual-resources" class="section level2">
+<h2><span class="header-section-number">2.2</span> Individual resources</h2>
+<p>Individual objects of a standard class can be added to the hub by providing only the data and metadata files or by creating a package as described in the <code>Family of Resources</code> section.</p>
+<p>OrgDb, TxDb and BSgenome objects are well defined <code>Bioconductor</code> classes and methods to download and process these objects already exist in <code>AnnotationHub</code>. When adding only one or two objects the overhead of creating a package may be unnecessary. The goal of the package is to provide structure for metadata generation and makes sense when there are plans to update versions or add new organisms in the future.</p>
+<p>Make sure the OrgDb, TxDb or BSgenome object you want to add does not already exist in the<br />
+<a href="http://www.bioconductor.org/packages/release/BiocViews.html#___AnnotationData">Biocondcutor annotation repository</a></p>
+<p>Providing just data and metadata files involves the following steps:</p>
+<ol style="list-style-type: decimal">
+<li><p>Notify <code>Bioconductor</code> team member:<br />
+Adding the data to AWS S3 and the metadata to the production database involves assistance from a <code>Bioconductor</code> team member. Please read the section “Uploading Data to S3”.</p></li>
+<li><p>Prepare the data:<br />
+In the case of an OrgDb object, only the sqlite file is stored in S3. See makeOrgPackageFromNCBI() and makeOrgPackage() in the <code>AnnotationForge</code> package for help creating the sqlite file. BSgenome objects should be made according to the steps outline in the <a href="http://www.bioconductor.org/packages/3.4/bioc/vignettes/BSgenome/inst/doc/BSgenomeForge.pdf">BSgenome vignette</a>. TxDb objects will be made on-the-fly from a GRanges with GenomicFeatures::makeTxDbFromGRanges() wh [...]
+<li><p>Generate metadata:<br />
+Prepare a .R file that generates metadata for the resource(s) by calling the <code>AnnotationHubData::AnnotationHubMetadata()</code> constructor. Argument details are found on the ?<code>AnnotationHubMetadata</code> man page.</p></li>
+</ol>
+<p>As an example, this piece of code generates the metadata for the Vitis vinifera TxDb Timothée Flutre contributed to <code>AnnotationHub</code>:</p>
+<pre class="r"><code>metadata <- AnnotationHubMetadata(
+ Description="Gene Annotation for Vitis vinifera",
+ Genome="IGGP12Xv0",
+ Species="Vitis vinifera",
+ SourceUrl="http://genomes.cribi.unipd.it/DATA/V2/V2.1/V2.1.gff3",
+ SourceLastModifiedDate=as.POSIXct("2014-04-17"),
+ SourceVersion="2.1",
+ RDataPath="community/tflutre/",
+ TaxonomyId=29760L,
+ Title="Vvinifera_CRIBI_IGGP12Xv0_V2.1.gff3.Rdata",
+ BiocVersion=package_version("3.3"),
+ Coordinate_1_based=TRUE,
+ DataProvider="CRIBI",
+ Maintainer="Timothée Flutre <timothee.flutre at supagro.inra.fr",
+ RDataClass="GRanges",
+ DispatchClass="GRanges",
+ SourceType="GFF",
+ RDataDateAdded=as.POSIXct(Sys.time()),
+ Recipe=NA_character_,
+ PreparerClass="None",
+ Tags=c("GFF", "CRIBI", "Gene", "Transcript", "Annotation"),
+ Notes="chrUn renamed to chrUkn"
+)</code></pre>
+<ol start="4" style="list-style-type: decimal">
+<li>Add data to S3 and metadata to the database:<br />
+This last step is done by the <code>Biocondcutor</code> team member.</li>
+</ol>
+</div>
+</div>
+<div id="additional-resources" class="section level1">
+<h1><span class="header-section-number">3</span> Additional resources</h1>
+<p>Metadata for new versions of the data can be added to the same package as they become available.</p>
+<ul>
+<li><p>The titles for the new versions should be unique and not match the title of any resource currently in AnnotationHub. Good practice would be to include the version and / or genome build in the title. If the title is not unique, the <code>AnnotationHub</code> object will list multiple files with the same title. The user will need to use ‘rdatadateadded’ to determine which is the most current.</p></li>
+<li><p>Make data available: see section on “Uploading Data to S3”</p></li>
+<li><p>Update make-metadata.R with the new metadata information</p></li>
+<li><p>Generate a new metadata.csv file. The package should contain metadata for all versions of the data in AnnotationHub so the old file should remain. When adding a new version it might be helpful to write a new csv file named by version, e.g., metadata_v84.csv, metadata_85.csv etc.</p></li>
+<li><p>Bump package version and commit to git</p></li>
+<li><p>Notify <a href="mailto:Lori.Shepherd at Roswellpark.org">Lori.Shepherd at Roswellpark.org</a> that an update is ready and a team member will add the new metadata to the production database; new resources will not be visible in AnnotationHub until the metadata are added to the database.</p></li>
+</ul>
+<p>Contact <a href="mailto:Lori.Shepherd at roswellpark.org">Lori.Shepherd at roswellpark.org</a> or <a href="mailto:maintainer at bioconductor.org">maintainer at bioconductor.org</a> with any questions.</p>
+</div>
+<div id="bug-fixes" class="section level1">
+<h1><span class="header-section-number">4</span> Bug fixes</h1>
+<p>A bug fix may involve a change to the metadata, data resource or both.</p>
+<div id="update-the-resource" class="section level2">
+<h2><span class="header-section-number">4.1</span> Update the resource</h2>
+<ul>
+<li><p>The replacement resource must have the same name as the original and be at the same location (path).</p></li>
+<li><p>Notify <a href="mailto:Lori.Shepherd at roswellpark.org">Lori.Shepherd at roswellpark.org</a> that you want to replace the data and make the files available: see section “Uploading Data to S3”.</p></li>
+</ul>
+</div>
+<div id="update-the-metadata" class="section level2">
+<h2><span class="header-section-number">4.2</span> Update the metadata</h2>
+<p>New metadata records can be added for new resources but modifying existing records is discouraged. Record modification will only be done in the case of bug fixes.</p>
+<ul>
+<li><p>Notify <a href="mailto:Lori.Shepherd at roswellpark.org">Lori.Shepherd at roswellpark.org</a> that you want to change the metadata</p></li>
+<li><p>Update make-metadata.R and regenerate the metadata.csv file</p></li>
+<li><p>Bump the package version and commit to git</p></li>
+</ul>
+</div>
+</div>
+<div id="remove-resources" class="section level1">
+<h1><span class="header-section-number">5</span> Remove resources</h1>
+<p>When a resource is removed from <code>AnnotationHub</code> two things happen: the ‘rdatadateremoved’ field is populated with a date and the ‘status’ field is populated with a reason why the resource is no longer available. Once these changes are made, the <code>AnnotationHub()</code> constructor will not list the resource among the available ids. An attempt to extract the resource with ‘[[’ and the AH id will return an error along with the status message.</p>
+<p>In general, resources are only removed when they are no longer available (e.g., moved from web location, no longer provided etc.).</p>
+<p>To remove a resource from <code>AnnotationHub</code> contact <a href="mailto:Lori.Shepherd at roswellpark.org">Lori.Shepherd at roswellpark.org</a> or <a href="mailto:maintainer at bioconductor.org">maintainer at bioconductor.org</a>.</p>
+</div>
+<div id="versioning" class="section level1">
+<h1><span class="header-section-number">6</span> Versioning</h1>
+<p>Versioning of resources is handled by the maintainer. If you plan to provide incremental updates to a file for the same organism / genome build, we recommend including a version in the title of the resource so it is easy to distinguish which is most current.</p>
+<p>If you do not include a version, or make the title unique in some way, multiple files with the same title will be listed in the <code>AnnotationHub</code> object. The user will can use the ‘rdatadateadded’ metadata field to determine which file is the most current.</p>
+</div>
+<div id="visibility" class="section level1">
+<h1><span class="header-section-number">7</span> Visibility</h1>
+<p>Several metadata fields control which resources are visible when a user invokes AnnotationHub(). Records are filtered based on these criteria:</p>
+<ul>
+<li>‘snapshotdate’ >= the date of the Bioconductor release being used</li>
+<li>‘rdatadateadded’ >= today’s date</li>
+<li>‘rdatadateremoved’ is NULL / NA</li>
+<li>‘biocVersion’ is <= to the Bioconductor version being used</li>
+</ul>
+<p>Once a record is added to AnnotationHub it is visable from that point forward until stamped with ‘rdatadateremoved’. For example, a record added on May 1, 2017 with ‘biocVersion’ 3.6 will be visible in all snapshots >= May1, 2017 and in all Bioconductor versions >= 3.6.</p>
+</div>
+<div id="uploading-data-to-s3" class="section level1">
+<h1><span class="header-section-number">8</span> Uploading Data to S3</h1>
+<p>Instead of providing the data files via dropbox, ftp, etc. we will grant temporary access to an S3 bucket where you can upload your data. Please email <a href="mailto:Lori.Shepherd at roswellpark.org">Lori.Shepherd at roswellpark.org</a> for access.</p>
+<p>You will be given access to the ‘AnnotationContributor’ user. Ensure that the <code>AWS CLI</code> is installed on your machine. See instructions for installing <code>AWS CLI</code> <a href="https://aws.amazon.com/cli/">here</a>. Once you have requested access you will be emailed a set of keys. There are two options to set the profile up for AnnotationContributor</p>
+<ol style="list-style-type: decimal">
+<li>Update your <code>.aws/config</code> file to include the following updating the keys accordingly:</li>
+</ol>
+<pre><code>[profile AnnotationContributor]
+output = text
+region = us-east-1
+aws_access_key_id = ****
+aws_secret_access_key = ****</code></pre>
+<ol start="2" style="list-style-type: decimal">
+<li>If you can’t find the <code>.aws/config</code> file, Run the following command entering appropriate information from above</li>
+</ol>
+<pre><code>aws configure --profile AnnotationContributor</code></pre>
+<p>After the configuration is set you should be able to upload resources using</p>
+<pre><code>aws --profile AnnotationContributor s3 cp test_file.txt s3://annotation-contributor/test_file.txt --acl public-read
+</code></pre>
+<p>Please upload the data with the appropriate directory structure, including subdirectories as necessary (i.e. top directory must be software package name, then if applicable, subdirectories of versions, …)</p>
+<p>Once the upload is complete, email <a href="mailto:Lori.Shepherd at roswellpark.org">Lori.Shepherd at roswellpark.org</a> to continue the process</p>
+</div>
+
+
+
+
+</div>
+
+<script>
+
+// add bootstrap table styles to pandoc tables
+function bootstrapStylePandocTables() {
+ $('tr.header').parent('thead').parent('table').addClass('table table-condensed');
+}
+$(document).ready(function () {
+ bootstrapStylePandocTables();
+});
+
+
+</script>
+
+<script type="text/x-mathjax-config">
+ MathJax.Hub.Config({
+ "HTML-CSS": {
+ styles: {
+ ".MathJax_Display": {
+ "text-align": "center",
+ padding: "0px 150px 0px 65px",
+ margin: "0px 0px 0.5em"
+ },
+ }
+ }
+ });
+</script>
+<!-- dynamically load mathjax for compatibility with self-contained -->
+<script>
+ (function () {
+ var script = document.createElement("script");
+ script.type = "text/javascript";
+ script.src = "https://mathjax.rstudio.com/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML";
+ document.getElementsByTagName("head")[0].appendChild(script);
+ })();
+</script>
+
+</body>
+</html>
diff --git a/man/AnnotationHub-class.Rd b/man/AnnotationHub-class.Rd
index 0b63141..f54cf78 100644
--- a/man/AnnotationHub-class.Rd
+++ b/man/AnnotationHub-class.Rd
@@ -89,7 +89,9 @@
immediate use.
Use \code{AnnotationHub()} to retrieve information about all records
- in the hub.
+ in the hub. If working offline, add argument \code{localHub=TRUE} to
+ work with a local, non-updated hub; It will only have resources
+ available that have previously been downloaded.
Discover records in a hub using \code{mcols()}, \code{query()},
\code{subset()}, \code{[}, and \code{display()}.
@@ -115,7 +117,8 @@
\item{}{
\code{AnnotationHub(..., hub=getAnnotationHubOption("URL"),
cache=getAnnotationHubOption("CACHE"),
- proxy=getAnnotationHubOption("PROXY"))}:
+ proxy=getAnnotationHubOption("PROXY"),
+ localHub=FALSE)}:
Create an \code{AnnotationHub} instance, possibly updating the
current database of records.
@@ -239,7 +242,9 @@
describing how matches across pattern elements are to be
combined. The default \code{`&`} requires that only records
with \emph{all} elements of \code{pattern} in their metadata
- columns are returned.}
+ columns are returned. \code{`&`}, \code{`|`} and \code{`!`}
+ are most notably available. See \code{"?&"} or
+ \code{?base::Ops} for more information.}
}
}
\item{}{
@@ -249,7 +254,7 @@
expression can reference columns of \code{mcols(x)}, and should
return a logical vector of length \code{length(x)}.
e.g., \code{subset(x, species == "Homo sapiens" &
- genome="GRCh38")}.
+ genome=="GRCh38")}.
}
\item{}{
\code{display(object)}:
@@ -299,14 +304,6 @@
calling this function, the user will have to download any AnnotationHub
resources again.
}
- \item{}{
- \code{getAnnotationHubOption()}:
- TODO: Get cache options "CACHE", "URL", "MAXDOWNLOADS" ...
- }
- \item{}{
- \code{setAnnotationHubOption()}:
- TODO: Set cache options "CACHE", "URL", "MAXDOWNLOADS" ...
- }
}
}
diff --git a/man/AnnotationHub-package.Rd b/man/AnnotationHub-package.Rd
index cd725ce..aa2fbed 100644
--- a/man/AnnotationHub-package.Rd
+++ b/man/AnnotationHub-package.Rd
@@ -3,12 +3,16 @@
\docType{package}
\title{Light-weight AnnotationHub 3.0 Client}
\description{
- Client to Bioconductor AnnotationHub 3.0 for discovery and retrieval
- of annotation resources.
+ Client for discovery and retrieval of Bioconductor annotation resources.
}
\author{Martin Morgan \url{mtmorgan at fhcrc.org}}
\keyword{ package }
+\seealso{\code{AnnotationHub-class}}
\examples{
-packageDescription("AnnotationHub")
+\dontrun{
+library(AnnotationHub)
+hub = AnnotationHub()
+hub
+}
}
diff --git a/man/AnnotationHubResource-class.Rd b/man/AnnotationHubResource-class.Rd
index a2ca0c1..baaa3c1 100644
--- a/man/AnnotationHubResource-class.Rd
+++ b/man/AnnotationHubResource-class.Rd
@@ -8,45 +8,24 @@
% Constructor
% Accessor-like methods
-\alias{getHub}
-\alias{getHub,AnnotationHubResource-method}
-\alias{hubCache,AnnotationHubResource-method}
-\alias{hubUrl,AnnotationHubResource-method}
+\alias{getHub}
+\alias{getHub,AnnotationHubResource-method}
+\alias{hubCache,AnnotationHubResource-method}
+\alias{hubUrl,AnnotationHubResource-method}
\title{AnnotationHubResource objects and their related methods and functions}
\description{
- TODO
-}
-
-\section{Accessors}{
- In the code snippets below, \code{x} and \code{object} are
- AnnotationHubResource objects.
-
- \describe{
- \item{}{
- \code{getHub(x)}:
- Gets the AnnotationHub from the hub slot.
- }
- \item{}{
- \code{getCache(x)}:
- Gets the location of the AnnotationHub cache from the AnnotationHub
- object in the hub slot.
- }
- \item{}{
- \code{getUrl(x)}:
- Gets the location of the AnnotationHub production server from the
- AnnotationHub object in the hub slot.
- }
- }
+ This class is utilized internally. AnnotationHubResources are
+ created as an intermediate 'on the fly' process when loading a
+ resource. It's accessor methods are used internally.
}
\author{Bioconductor Core Team}
-\examples{
- ## TODO
-}
+\seealso{\link{AnnotationHub-class}}
+\keyword{internal}
\keyword{classes}
\keyword{methods}
diff --git a/man/listResources.Rd b/man/listResources.Rd
index 20a40ed..b51c77f 100644
--- a/man/listResources.Rd
+++ b/man/listResources.Rd
@@ -2,9 +2,7 @@
\alias{utilities}
\alias{listResources}
-\alias{listResources,AnnotationHub-method}
\alias{loadResources}
-\alias{loadResources,AnnotationHub-method}
\title{
@@ -13,59 +11,13 @@
\description{
List and load resources from ExperimentHub filtered by package
- name and optional search terms.
+ name and optional search terms. Not Implemented for
+ AnnotationHub.
}
\details{
Currently \code{listResources} and \code{loadResources} are only meaningful
- for \code{ExperimentHub} objects. Methods for \code{AnnotationHub}
- objects may be added in the future.
+ for \code{ExperimentHub} objects.
}
-\usage{
-listResources(hub, package, filterBy = character())
-loadResources(hub, package, filterBy = character())
-}
-
-\arguments{
- \item{hub}{
- A \code{Hub} object; currently only meaningful for \code{ExperimentHub}.
- }
- \item{package}{
- A \code{character(1)} name of a package with resources hosted in the Hub.
- }
- \item{filterBy}{
- A \code{character()} vector of search terms for additional filtering.
- Can be any terms found in the metadata (mcols()) of the resources.
- When not provided, there is no additional filtering and all resources
- associated with the given package are returned.
- }
-}
-
-\value{
- \code{listResources} returns a character vector;
- \code{loadResources} returns a list of data objects.
-}
-
-\seealso{
-}
-
-\examples{
-\dontrun{
-## Packages with resources hosted in ExperimentHub:
-require(ExperimentHub)
-eh <- ExperimentHub()
-unique(package(eh))
-
-## All resources associated with the 'GSE62944' package:
-listResources(eh, "GSE62944")
-
-## Resources associated with the 'curatedMetagenomicData' package
-## filtered by 'plaque.abundance':
-listResources(eh, "curatedMetagenomicData", "plaque.abundance")
-
-## 'loadResources()' returns a list of the data objects:
-loadResources(eh, "curatedMetagenomicData", "plaque.abundance")
-}
-}
\keyword{utilities}
diff --git a/man/readMetadataFromCsv.Rd b/man/readMetadataFromCsv.Rd
new file mode 100644
index 0000000..82a1135
--- /dev/null
+++ b/man/readMetadataFromCsv.Rd
@@ -0,0 +1,194 @@
+\name{readMetadataFromCsv}
+
+\alias{readMetadataFromCsv}
+\alias{metadata.csv}
+
+\title{
+ Reads csv files of resource metadata into a data.frame
+}
+
+\description{
+ Reads metadata files located in the "inst/extdata" package directory
+ into a data.frame.
+}
+
+\usage{
+ readMetadataFromCsv(pathToPackage, fileName=character())
+}
+
+\arguments{
+ \item{pathToPackage}{
+ Full path to data package including package name; no trailing slash.
+ }
+ \item{fileName}{
+ Name of single metadata file located in "inst/extdata". If none is
+ provided the function looks for a file named "metadata.csv".
+ }
+}
+
+\details{
+ \itemize{
+ \item{readMetadataFromCsv:}{
+
+ Reads a .csv file of metadata located in "inst/extdata".
+ \code{readMetadataFromCsv} performs checks for required columns
+ and data types and can be used by package authors to validate their
+ metadata before submitting the package for review. The function is
+ used internally by \code{AnnotationHubData::makeAnnotationHubMetadata}.
+
+ The rows of the .csv file(s) represent individual \code{Hub}
+ resources (i.e., data objects) and the columns are the metadata
+ fields. All fields should be a single character string of length 1.
+
+ Required Fields in metadata file:
+ \itemize{
+ \item Title: \code{character(1)}. Name of the resource. This can be
+ the exact file name (if self-describing) or a more complete
+ description.
+
+ \item Description: \code{character(1)}. Brief description of the
+ resource, similar to the 'Description' field in a package
+ DESCRIPTION file.
+
+ \item BiocVersion: \code{character(1)}. The first Bioconductor version
+ the resource was made available for. Unless removed from
+ the hub, the resource will be available for all versions
+ greater than or equal to this field.
+
+ \item Genome: \code{character(1)}. Genome.
+
+ \item SourceType: \code{character(1)}. Format of original data, e.g., FASTA,
+ BAM, BigWig, etc.
+
+ \item SourceUrl: \code{character(1)}. Optional location of original
+ data files. Multiple urls should be provided as a comma separated
+ string.
+
+ \item SourceVersion: \code{character(1)}. Version of original data.
+
+ \item Species: \code{character(1)}. Species.
+
+ \item TaxonomyId: \code{character(1)}. Taxonomy ID.
+
+ \item Coordinate_1_based: \code{logical}. TRUE if data are 1-based.
+
+ \item DataProvider: \code{character(1)}. Name of company or institution
+ that supplied the original (raw) data.
+
+ \item Maintainer: \code{character(1)}. Maintainer name and email in the
+ following format: Maintainer Name <username at address>.
+
+ \item RDataClass: \code{character(1)}. R / Bioconductor class the data
+ are stored in, e.g., GRanges, SummarizedExperiment,
+ ExpressionSet etc.
+
+ \item DispatchClass: \code{character(1)}. Determines how data are
+ loaded into R. The value for this field should be
+ \sQuote{Rda} if the data were serialized with \code{save()} and
+ \sQuote{Rds} if serialized with \code{saveRDS}. The filename
+ should have the appropriate \sQuote{rda} or \sQuote{rds}
+ extension.
+
+ A number of dispatch classes are pre-defined in
+ AnnotationHub/R/AnnotationHubResource-class.R with the suffix
+ \sQuote{Resource}. For example, if you have sqlite files, the
+ AnnotationHubResource-class.R defines SQLiteFileResource so
+ the DispatchClass would be SQLiteFile. Contact
+ maintainer at bioconductor.org if you are not sure which class
+ to use.
+
+ \item Location_Prefix: \code{character(1)}. Do not include this field
+ if data are stored in AWS S3; it will be generated automatically.
+
+ If data will be accessed from a location other than AWS S3
+ this field should be the base url.
+
+ \item RDataPath: \code{character(1)}.This field should be the
+ remainder of the path to the resource. The
+ \code{Location_Prefix} will be prepended to
+ \code{RDataPath} for the full path to the resource.
+ If the resource is stored in Bioconductor's AWS S3
+ buckets, it should start with the name of the package associated
+ with the metadata and should not start with a leading
+ slash. It should include the resource file name.
+
+ \item Tags: \code{character() vector}.
+ \sQuote{Tags} are search terms used to define a subset of
+ resources in a \code{Hub} object, e.g, in a call to \code{query}.
+
+ For ExperimentHub resources, \sQuote{Tags} are automatically
+ generated from the \sQuote{biocViews} in the DESCRIPTION.
+ \sQuote{Tags} values supplied by the user are not be entered in
+ the database and are not part of the formal metadata. This
+ 'controlled vocabulary' approach was taken to limit the search
+ terms to a well defined set and may change in the future.
+
+ \sQuote{Tags} for AnnotationHub resources are a free-form field
+ of search terms defined by the user. The package name is added
+ as one of the \sQuote{Tags} before the metadata are finalized.
+ Multiple \sQuote{Tags} are specified as a colon separated
+ string, e.g., tags for two resources would look like this:
+
+ \preformatted{
+ Tags=c("tag1:tag2:tag3", "tag1:tag3")
+ }
+ }
+ NOTE: The metadata file can have additional columns beyond the 'Required
+ Fields' listed above. These values are not added to the Hub database but
+ they can be used in package functions to provide an additional level of
+ metadata on the resources.
+ }
+ }
+}
+
+\value{
+ A data.frame with one row per resource and columns for the Required
+ Fields described above. Additional auto-generated columns, e.g.,
+ RDataDateAdded and PreparerClass may also be present and are
+ used by internal functions when generating the final metadata.
+}
+
+\seealso{
+ \itemize{
+ \item \code{\link[AnnotationHubData]{makeAnnotationHubMetadata}}
+ }
+}
+
+\examples{
+
+## Each row of the metadata file represents a resource added to one of
+## the 'Hubs'. This example creates a metadata.csv file for a single resource.
+## In the case of multiple resources, the arguments below would be character
+## vectors that produced multiple rows in the data.frame.
+
+meta <- data.frame(
+ Title = "RNA-Sequencing dataset from study XYZ",
+ Description = paste0("RNA-seq data from study XYZ containing 10 normal ",
+ "and 10 tumor samples represented as a",
+ "SummarizedExperiment"),
+ BiocVersion = "3.4",
+ Genome = "GRCh38",
+ SourceType = "BAM",
+ SourceUrl = "http://www.path/to/original/data/file",
+ SourceVersion = "Jan 01 2016",
+ Species = "Homo sapiens",
+ TaxonomyId = 9606,
+ Coordinate_1_based = TRUE,
+ DataProvider = "GEO",
+ Maintainer = "Your Name <youremail at provider.com>",
+ RDataClass = "SummarizedExperiment",
+ DispatchClass = "Rda",
+ ResourceName = "FileName.rda"
+)
+
+\dontrun{
+## Write the data out and put in the inst/extdata directory.
+write.csv(meta, file="metadata.csv", row.names=FALSE)
+
+## Test the validity of metadata.csv with readMetadataCsv():
+readMetadataFromCsv("path/to/mypackage")
+}
+
+}
+
+\keyword{methods}
diff --git a/vignettes/AnnotationHub-HOWTO.Rmd b/vignettes/AnnotationHub-HOWTO.Rmd
index 953199a..2bbec22 100644
--- a/vignettes/AnnotationHub-HOWTO.Rmd
+++ b/vignettes/AnnotationHub-HOWTO.Rmd
@@ -307,8 +307,8 @@ evidence of medical interest. This information is available at
Query the dbDNP files in the hub:
```{r echo=FALSE, results='hide', message=FALSE}
-query(ah, c("GRCh37", "dbSNP", "VCF" ))
-vcf <- ah[['AH50424']]
+query(ah, c("GRCh38", "dbSNP", "VCF" ))
+vcf <- ah[['AH57960']]
```
This returns a _VcfFile_ which can be read in using `r
Biocpkg("VariantAnnotation")`; because VCF files can be large, `readVcf()`
diff --git a/vignettes/CreateAnAnnotationPackage.Rmd b/vignettes/CreateAnAnnotationPackage.Rmd
new file mode 100644
index 0000000..51fa8da
--- /dev/null
+++ b/vignettes/CreateAnAnnotationPackage.Rmd
@@ -0,0 +1,354 @@
+---
+title: "Creating An AnnotationHub Package"
+author: "Valerie Obenchain and Lori Shepherd"
+date: "Modified: October 2016. Compiled: `r format(Sys.Date(), '%d %b %Y')`"
+output:
+ BiocStyle::html_document:
+ toc: true
+vignette: >
+ % \VignetteIndexEntry{AnnotationHub: Creating An AnnotationHub Package}
+ % \VignetteEngine{knitr::rmarkdown}
+ % \VignetteEncoding{UTF-8}
+---
+
+
+# Overview
+
+The `AnnotationHubData` package provides tools to acquire, annotate, convert
+and store data for use in Bioconductor's `AnnotationHub`. BED files from the
+Encode project, gtf files from Ensembl, or annotation tracks from UCSC, are
+examples of data that can be downloaded, described with metadata, transformed
+to standard `Bioconductor` data types, and stored so that they may be
+conveniently served up on demand to users via the AnnotationHub client. While
+data are often manipulated into a more R-friendly form, the data themselves
+retain their raw content and are not filtered or curated like those in
+[ExperimentHub](http://bioconductor.org/packages/ExperimentHub/).
+Each resource has associated metadata that can be searched through the
+`AnnotationHub` client interface.
+
+# New resources
+
+## Family of resources
+
+Multiple, related resources are added to `AnnotationHub` by creating a software
+package similar to the existing annotation packages. The package itself does
+not contain data but serves as a light weight wrapper around scripts that
+generate metadata for the resources added to `AnnotationHub`.
+
+At a minimum the package should contain a man page describing the resources.
+Vignettes and additional `R` code for manipulating the objects are optional.
+
+Creating the package involves the following steps:
+
+1. Notify `Bioconductor` team member:
+ Man page and vignette examples in the software package will not work until
+ the data are available in `AnnotationHub`. Adding the data to AWS S3 and the
+ metadata to the production database involves assistance from a `Bioconductor`
+ team member. Please read the section "Uploading Data to S3".
+
+2. Building the software package:
+ Below is an outline of package organization. The files listed are required
+ unless otherwise stated.
+
+* inst/extdata/
+
+ - metadata.csv:
+ This file contains the metadata in the format of one row per resource
+ to be added to the `AnnotationHub` database. The file should be generated
+ from the code in inst/scripts/make-metadata.R where the final data are
+ written out with write.csv(..., row.names=FALSE). The required column
+ names and data types are specified in
+ `AnnotationHub::readMetadataFromCsv()`. See ?`readMetadataFromCsv` for
+ details.
+
+ If necessary, metadata can be broken up into multiple csv files instead
+ having of all records in a single "metadata.csv".
+
+* inst/scripts/
+
+ - make-data.R:
+ A script describing the steps involved in making the data object(s). This
+ includes where the original data were downloaded from, pre-processing, and
+ how the final R object was made. Include a description of any steps
+ performed outside of `R` with third party software. Output of the script
+ should be files on disk ready to be pushed to S3. If data are to be hosted
+ on a personal web site instead of S3, this file should explain any
+ manipulation of the data prior to hosting on the web site. For data hosted
+ on a public web site with no prior manipultaion this file is not needed.
+
+ - make-metadata.R:
+ A script to make the metadata.csv file located in inst/extdata of the
+ package. See ?`readMetadataFromCsv` for a description of the
+ metadata.csv file, expected fields and data types. The
+ `readMetadataFromCsv()` function can be used to validate the metadata.csv
+ file before submitting the package.
+
+* vignettes/
+
+ OPTIONAL vignette(s) describing analysis workflows.
+
+* R/
+
+ OPTIONAL functions to enhance data exploration.
+
+* man/
+
+ - package man page:
+ OPTIONAL. The package man page serves as a landing point and should briefly
+ describe all resources associated with the package. There should be an
+ \alias entry for each resource title either on the package man page or
+ individual man pages.
+
+ - resource man pages:
+ OPTIONAL. Man page(s) should describe the resource (raw data source,
+ processing, QC steps) and demonstrate how the data can be loaded through
+ the `AnnotationHub` interface. For example, replace "SEARCHTERM*" below
+ with one or more search terms that uniquely identify resources in your
+ package.
+
+ ```
+ library(AnnotationHub)
+ hub <- AnnotationHub()
+ myfiles <- query(hub, "SEARCHTERM1", "SEARCHTERM2")
+ myfiles[[1]] ## load the first resource in the list
+ ```
+
+* DESCRIPTION / NAMESPACE
+The scripts used to generate the metadata will likely use functions from
+AnnotationHub or AnnotationHubData which should be listed in Depends/Imports as
+necessary.
+
+
+3. Data objects:
+Data are not formally part of the software package and are stored
+separately in AWS S3 buckets. The author should follow instructions in the
+section "Uploading Data to S3"
+
+4. Confirm valid metadata:
+Confirm the data in inst/exdata/metadata.csv are valid by running
+AnnotationHubData:::makeAnnotationHubMetadata() on your package. Please
+address and warnings or errors.
+
+5. Package review:
+Submit the package to the
+[tracker](https://github.com/Bioconductor/Contributions) for review. The
+primary purpose of the package review is to validate the metadata in the csv
+file(s). It is ok if the package fails R CMD build and check because the
+data and metadata are not yet in place. Once the metadata.csv is approved,
+records are added to the production database. At that point the package man
+pages and vignette can be finalized and the package should pass R CMD build
+and check.
+
+## Individual resources
+
+Individual objects of a standard class can be added to the hub by providing
+only the data and metadata files or by creating a package as described in the
+`Family of Resources` section.
+
+OrgDb, TxDb and BSgenome objects are well defined `Bioconductor` classes and
+methods to download and process these objects already exist in `AnnotationHub`.
+When adding only one or two objects the overhead of creating a package may be
+unnecessary. The goal of the package is to provide structure for metadata
+generation and makes sense when there are plans to update versions or add new
+organisms in the future.
+
+Make sure the OrgDb, TxDb or BSgenome object you want to add does not already
+exist in the
+[Biocondcutor annotation repository](http://www.bioconductor.org/packages/release/BiocViews.html#___AnnotationData)
+
+Providing just data and metadata files involves the following steps:
+
+1. Notify `Bioconductor` team member:
+ Adding the data to AWS S3 and the metadata to the production database
+ involves assistance from a `Bioconductor` team member. Please read the
+ section "Uploading Data to S3".
+
+2. Prepare the data:
+ In the case of an OrgDb object, only the sqlite file is stored in S3.
+ See makeOrgPackageFromNCBI() and makeOrgPackage() in the `AnnotationForge`
+ package for help creating the sqlite file. BSgenome objects should be made
+ according to the steps outline in the
+ [BSgenome
+ vignette](http://www.bioconductor.org/packages/3.4/bioc/vignettes/BSgenome/inst/doc/BSgenomeForge.pdf). TxDb objects will be made on-the-fly from a
+ GRanges with GenomicFeatures::makeTxDbFromGRanges() when the resource is
+ downloaded from `AnnotationHub`. Data should be provided as a GRanges
+ object. See GenomicRanges::makeGRangesFromDataFrame() or
+ rtracklayer::import() for help creating the GRanges.
+
+3. Generate metadata:
+ Prepare a .R file that generates metadata for the resource(s) by calling
+ the `AnnotationHubData::AnnotationHubMetadata()` constructor. Argument
+ details are found on the ?`AnnotationHubMetadata` man page.
+
+ As an example, this piece of code generates the metadata for
+ the Vitis vinifera TxDb Timothée Flutre contributed to `AnnotationHub`:
+
+```{r, TxDb_Metadata, eval=FALSE}
+metadata <- AnnotationHubMetadata(
+ Description="Gene Annotation for Vitis vinifera",
+ Genome="IGGP12Xv0",
+ Species="Vitis vinifera",
+ SourceUrl="http://genomes.cribi.unipd.it/DATA/V2/V2.1/V2.1.gff3",
+ SourceLastModifiedDate=as.POSIXct("2014-04-17"),
+ SourceVersion="2.1",
+ RDataPath="community/tflutre/",
+ TaxonomyId=29760L,
+ Title="Vvinifera_CRIBI_IGGP12Xv0_V2.1.gff3.Rdata",
+ BiocVersion=package_version("3.3"),
+ Coordinate_1_based=TRUE,
+ DataProvider="CRIBI",
+ Maintainer="Timothée Flutre <timothee.flutre at supagro.inra.fr",
+ RDataClass="GRanges",
+ DispatchClass="GRanges",
+ SourceType="GFF",
+ RDataDateAdded=as.POSIXct(Sys.time()),
+ Recipe=NA_character_,
+ PreparerClass="None",
+ Tags=c("GFF", "CRIBI", "Gene", "Transcript", "Annotation"),
+ Notes="chrUn renamed to chrUkn"
+)
+```
+
+4. Add data to S3 and metadata to the database:
+ This last step is done by the `Biocondcutor` team member.
+
+# Additional resources
+
+Metadata for new versions of the data can be added to the same package as they
+become available.
+
+* The titles for the new versions should be unique and not match the title of
+ any resource currently in AnnotationHub. Good practice would be to
+ include the version and / or genome build in the title. If the title is
+ not unique, the `AnnotationHub` object will list multiple files with the
+ same title. The user will need to use 'rdatadateadded' to determine which
+ is the most current.
+
+* Make data available: see section on "Uploading Data to S3"
+
+* Update make-metadata.R with the new metadata information
+
+* Generate a new metadata.csv file. The package should contain
+ metadata for all versions of the data in AnnotationHub so the old file should
+ remain. When adding a new version it might be helpful to write a new csv
+ file named by version, e.g., metadata_v84.csv, metadata_85.csv etc.
+
+* Bump package version and commit to git
+
+* Notify Lori.Shepherd at Roswellpark.org that an update is ready and
+ a team member will add the new metadata to the production database;
+ new resources will not be visible in AnnotationHub until
+ the metadata are added to the database.
+
+Contact Lori.Shepherd at roswellpark.org or maintainer at bioconductor.org with any
+questions.
+
+# Bug fixes
+
+A bug fix may involve a change to the metadata, data resource or both.
+
+## Update the resource
+
+* The replacement resource must have the same name as the original and
+ be at the same location (path).
+
+* Notify Lori.Shepherd at roswellpark.org that you want to replace the data
+ and make the files available: see section "Uploading Data to S3".
+
+## Update the metadata
+
+New metadata records can be added for new resources but modifying existing
+records is discouraged. Record modification will only be done in the case of
+bug fixes.
+
+* Notify Lori.Shepherd at roswellpark.org that you want to change the metadata
+
+* Update make-metadata.R and regenerate the metadata.csv file
+
+* Bump the package version and commit to git
+
+# Remove resources
+
+When a resource is removed from `AnnotationHub` two things happen:
+the 'rdatadateremoved' field is populated with a date and the 'status'
+field is populated with a reason why the resource is no longer available. Once
+these changes are made, the `AnnotationHub()` constructor will not list the
+resource among the available ids. An attempt to extract the resource with
+'[[' and the AH id will return an error along with the status message.
+
+In general, resources are only removed when they are no longer available
+(e.g., moved from web location, no longer provided etc.).
+
+To remove a resource from `AnnotationHub` contact Lori.Shepherd at roswellpark.org
+or maintainer at bioconductor.org.
+
+# Versioning
+
+Versioning of resources is handled by the maintainer. If you plan to provide
+incremental updates to a file for the same organism / genome build, we
+recommend including a version in the title of the resource so it is easy
+to distinguish which is most current.
+
+If you do not include a version, or make the title unique in some way,
+multiple files with the same title will be listed in the `AnnotationHub`
+object. The user will can use the 'rdatadateadded' metadata field
+to determine which file is the most current.
+
+# Visibility
+
+Several metadata fields control which resources are visible when
+a user invokes AnnotationHub(). Records are filtered based on these criteria:
+
+- 'snapshotdate' >= the date of the Bioconductor release being used
+- 'rdatadateadded' >= today's date
+- 'rdatadateremoved' is NULL / NA
+- 'biocVersion' is <= to the Bioconductor version being used
+
+Once a record is added to AnnotationHub it is visable from that point forward
+until stamped with 'rdatadateremoved'. For example, a record added on
+May 1, 2017 with 'biocVersion' 3.6 will be visible in all snapshots >=
+May1, 2017 and in all Bioconductor versions >= 3.6.
+
+# Uploading Data to S3
+
+Instead of providing the data files via dropbox, ftp, etc. we will grant
+temporary access to an S3 bucket where you can upload your data. Please
+email Lori.Shepherd at roswellpark.org for access.
+
+You will be given access to the 'AnnotationContributor' user. Ensure that the
+`AWS CLI` is installed on your machine. See instructions for installing `AWS
+CLI` [here](https://aws.amazon.com/cli/). Once you have requested access you
+will be emailed a set of keys. There are two options to set the profile up for
+AnnotationContributor
+
+1. Update your `.aws/config` file to include the following updating the keys
+accordingly:
+
+```
+[profile AnnotationContributor]
+output = text
+region = us-east-1
+aws_access_key_id = ****
+aws_secret_access_key = ****
+```
+2. If you can't find the `.aws/config` file, Run the following command entering
+appropriate information from above
+
+```
+aws configure --profile AnnotationContributor
+```
+
+After the configuration is set you should be able to upload resources using
+
+```
+aws --profile AnnotationContributor s3 cp test_file.txt s3://annotation-contributor/test_file.txt --acl public-read
+
+```
+
+Please upload the data with the appropriate directory structure, including
+subdirectories as necessary (i.e. top directory must be software package name,
+then if applicable, subdirectories of versions, ...)
+
+Once the upload is complete, email Lori.Shepherd at roswellpark.org to continue the
+process
+
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/debian-med/r-bioc-annotationhub.git
More information about the debian-med-commit
mailing list