*-------------------------------------------------------------------------------
* to support a new input file format (19 jan 2012) by example: narrowPeak

  1) find a short example file
     library(AnnotationHubData)
     importer <- EncodeImporter()
     tbl.md <- metadataTable(importer)
     nrow(subset(tbl.md, type=='narrowPeak')) # [1] 1193
     fivenum(subset(tbl.md, type=='narrowPeak')$size) #     536  578000  914000 1900000 5200000
     rownames(subset(tbl.md, type=='narrowPeak' & size==536)) # [1] 
         # "wgEncodeSydhTfbsK562Xrcc4StdPk.narrowPeak.gz"

  2) the metadata (collected from files.txt in each ucsc/encode project directory)
     helps to fill out much of the json-related, AnnotationHub-required
     per-file metadata (per-resource metadata)
     EncodeImporter::assembleParams does this job, assigning the fields
     of a params object which can be used this way in
     EncodeImporter::createResource:

         md <- do.call(AnnotationHubMetadata, params)

  3) every resource has a source directory, from which a gzipped data file is read
     and to which an AnnotationHubServer-ready RData file is written.
     An AnnotationHubRecipe specifies how the first is transformed to the
     second.  a json file can express those params.  a mongo datbase
     entry can too.  when adding a new encode data file format (here 'narrowPeak')
     we must make sure that the params successfully specify the transformation.

  4) test_AnnotationHubRecipe::test_narrowPeakToGranges is that function.
     it needs a directory with the gzipped data file, and a json file
         $(AHROOT)/inst/extdata/goldenpath/hg19/encodeDCC/wgEncodeSydhTfbs:
     where AHROOT today is /home/ubuntu/AnnotationHubData/

       1575 Jan 20 01:00 wgEncodeSydhTfbsK562Brf2StdPk.narrowPeak_0.0.1.json
        664 Jan 19 21:52 wgEncodeSydhTfbsK562Brf2StdPk.narrowPeak.gz

     the test consists of these commands:

          md <- constructMetadataFromJsonPath(annotationHubRoot, jsonPath)
          recipe <- AnnotationHubRecipe(md)
          checkEquals(recipeName(recipe), "extendedBedToGRanges")
          pathToRDataFile <- run(recipe)
          load(pathToRDataFile)
          checkEquals(length(gr), 19)
          checkEquals(names(mcols(gr)), c("name","score","signalValue","pValue","qValue", "peak"))

      the final step of EncodeImporter::createResource is
         stopifnot(file.exists(localJsonFile))
         stopifnot(json2mongo(localJsonFile))

  5) this illustrates that json files are not a permanent part of the AnnotationHub.
     But they are needed on import, and their contents have to WORK:  good GRanges
     (or whatever) need to result from running a recipe on the params

  6) so the creation of support for a new data format requires a json 
     file.  for that purpose, use

         ~/AnnotationHubData/inst/scripts/createJsonForTesting.R

      called this way:

          projectName <- "wgEncodeSydhTfbs"
          dataFileName <- "wgEncodeSydhTfbsK562Brf2StdPk.narrowPeak.gz"
          jsonPath <- createJson(projectName, dataFileName)
          stopifnot(file.exists(jsonPath))

      all the necessary specificity is learned from 
           tbl.md[dataFileName,] (see above)
           EncodeImporter::assembleParams format-specific code, branching on
           tbl.md[dataFileName, "type"] to learn what recipe to use, with what
           colclasses.

   then add unit test:
       test_narrowPeakToGRanges()
           workingDirectory <- AnnotationHubData:::.createWorkingDirectory(sourceDirectory)
           annotationHubRoot <- workingDirectory
               # locate the json metadata file
           jsonFile <- "wgEncodeSydhTfbsK562Brf2StdPk.narrowPeak_0.0.1.json"
           resourcePath <- "goldenpath/hg19/encodeDCC/wgEncodeSydhTfbs"
           jsonPath <- file.path(resourcePath, jsonFile)
           fullJsonPath <- file.path(annotationHubRoot, jsonPath)
           checkTrue(file.exists(fullJsonPath))
               # create a metadata object from this file
           md <- constructMetadataFromJsonPath(annotationHubRoot, jsonPath)
               # now create a Recipe instance
           recipe <- AnnotationHubRecipe(md)
           checkEquals(recipeName(recipe), "extendedBedToGRanges")
               # create GRanges from the extended bed file, save as RData where
               # instructed by the recipe
           pathToRDataFile <- run(recipe)
               # check the result
           load(pathToRDataFile)
           checkEquals(length(gr), 19)
           checkEquals(names(mcols(gr)), c("name","score","signalValue","pValue","qValue", "peak"))




*-------------------------------------------------------------------------------
* add the gtf format, 892 files (20 jan 2013)

  nrow(subset(tbl.md, type=="gtf")) # [1] 892
  fivenum(subset(tbl.md, type=="gtf")$size) # [1]   165000  1900000  4000000  7500000 38000000
  rownames(subset(tbl.md, type=="gtf" &size==165000)) # "wgEncodeCshlLongRnaSeqHmecCellPamGeneDeNovo.gtf.gz"

  mkdir inst/extdata/goldenpath/hg19/encodeDCC/wgEncodeCshlLongRnaSeq
  cd inst/extdata/goldenpath/hg19/encodeDCC/wgEncodeCshlLongRnaSeq
  http://hgdownload.cse.ucsc.edu/goldenpath/hg19/encodeDCC/wgEncodeCshlLongRnaSeq/
  curl -O http://hgdownload.cse.ucsc.edu/goldenpath/hg19/encodeDCC/wgEncodeCshlLongRnaSeq/wgEncodeCshlLongRnaSeqHmecCellPamGeneDeNovo.gtf.gz

   8556 lines, first two
    chr1	Cufflinks	gene	16523	17364	757	-	.	gene_id "chr1.21"; IDR "."; FPKM1 "38.7913"; FPKM2 ".";
    chr1	Cufflinks	gene	17925	29343	897	-	.	gene_id "chr1.25"; IDR "."; FPKM1 "157.038"; FPKM2 ".";

  x <- import('../extdata/goldenpath/hg19/encodeDCC/wgEncodeCshlLongRnaSeq/wgEncodeCshlLongRnaSeqHmecCellPamGeneDeNovo.gtf.gz')

  --- create a json metadata file
     source('../scripts/createJsonForTesting.R')
     # which has these concluding lines
       projectName <- "wgEncodeCshlLongRnaSeq"
       dataFileName <- "wgEncodeCshlLongRnaSeqHmecCellPamGeneDeNovo.gtf.gz"
       jsonPath <- createJson(projectName, dataFileName)
       printf("created new jsonfile: %s", jsonPath)
       stopifnot(file.exists(jsonPath))

   --- add a unit test

*-------------------------------------------------------------------------------
* gtf  format described: http://genome.ucsc.edu/FAQ/FAQformat.html#format4
    	 

   GTF (Gene Transfer Format) is a refinement to GFF that tightens the
   specification. The first eight GTF fields are the same as GFF. The
   group field has been expanded into a list of attributes. Each
   attribute consists of a type/value pair. Attributes must end in a
   semi-colon, and be separated from any following attribute by
   exactly one space.

   The attribute list must begin with the two mandatory attributes:

      gene_id value - A globally unique identifier for the genomic source of the sequence.
      transcript_id value - A globally unique identifier for the predicted transcript. 

   Example:

   Here is an example of the ninth field in a GTF data line:

      gene_id "Em:U62317.C22.6.mRNA"; transcript_id "Em:U62317.C22.6.mRNA"; exon_number 1

   The Genome Browser groups together GTF lines that have the same transcript_id value. It only looks at features of type exon and CDS.

   For more information on this format, see http://mblab.wustl.edu/GTF2.html. 

*-------------------------------------------------------------------------------
