index.html preprocessing.html

1 Genome annotation input

There are a few methods of importing annotation data into R. The ‘new’ method is to use the DBI interfaces provided by bioconductor. These were a bit more difficult for me to learn, but now that I have a general idea of how they work, I grudgingly admit that they are far superior.

1.1 OrganismDbi/OrgDb/TxDb

The mouse OrganismDbi interface is contained the library ‘Mus.musculus.’ It brings together the OrgDb and TxDb packages for the mouse: ‘org.Mm.eg.db’ and ‘TxDb.Mmusculus.UCSC.mm10.knownGene’ respectively. I am going to need to figure out how to integrate the miRNA annotations into them, but for the moment I will work just with them and hopefully do them in a way which is cleaner and clearer than before.

## orgDb packages are responsible for providing mapping to other databases,
## like GO/KEGG/Entrez/Ensembl/etc
tmp <- sm(library("org.Mm.eg.db"))
## In contrast, TxDb packages are responsible for providing information about
## each transcript, thus providing a link between the actual genome sequence
## and the transcripts encoded within.
tmp <- sm(library("TxDb.Mmusculus.UCSC.mm10.knownGene"))
## the Mus.musculus package actually loads the previous 2, but I want to be explicit.
tmp <- sm(library("Mus.musculus"))

## The following section gathers annotation data from the 2015 archive of ensembl.
## In this case, all the annotations are explicitly the mirbase data
## along with a subset of other ensembl fields.
ensembl_genes <- biomaRt::useEnsembl("ensembl", host="dec2015.archive.ensembl.org")
genes_datasets <- biomaRt::listDatasets(ensembl_genes)
mouse_genes <- biomaRt::useEnsembl(biomart="ensembl", dataset="mmusculus_gene_ensembl",
                                   host="dec2015.archive.ensembl.org")
mirna_attribs <- c("ensembl_gene_id", "ensembl_transcript_id", "description",
                  "external_gene_name", "mirbase_accession", "mirbase_id",
                  "mgi_transcript_name", "chromosome_name")
mouse_mirna_attribs <- biomaRt::listAttributes(mart=mouse_genes)
mm_mi_genes <- biomaRt::getBM(attributes=mirna_attribs, filters="biotype",
                              values="miRNA", mart=mouse_genes)
## The following is used by miRNAtab.Rmd
mm_mi_genes$ID <- paste0("chr", mm_mi_genes$chromosome_name, "_", mm_mi_genes$ensembl_gene_id)
rownames(mm_mi_genes) <- make.names(mm_mi_genes$ID, unique=TRUE)

library(mirbase.db)
chosen_df <- mm_mi_genes[, c("ensembl_gene_id","mirbase_id")]
chosen_df$fivep_accession <- ""
chosen_df$fivep_id <- ""
chosen_df$threep_accession <- ""
chosen_df$threep_id <- ""
mikey <- "mmu-mir-344h-1"
names <- rownames(chosen_df)
for (minum in 1:length(chosen_df$mirbase_id)) {
    mikey <- chosen_df$mirbase_id[[minum]]
    if (mikey == "") {
        next
    }
    mi_mappings <- try(get(mikey, mirbaseMATURE), silent=TRUE)
    if (class(mi_mappings) == "try-error") {
        next
    }
    mi_accessions <- mi_mappings@matureAccession
    mi_names <- mi_mappings@matureName
    name <- names[[minum]]
    chosen_df[name, "fivep_accession"] <- mi_accessions[1]
    chosen_df[name, "threep_accession"] <- mi_accessions[2]
    chosen_df[name, "fivep_id"] <- mi_names[1]
    chosen_df[name, "threep_id"] <- mi_names[2]
}
write.table(x=chosen_df, file="reference/mi_mappings.tab")

ensembl_new <- biomaRt::useEnsembl("regulation")
new_datasets <- biomaRt::listDatasets(ensembl_new)
mouse_dataset <- biomaRt::useEnsembl(biomart="regulation", dataset="mmusculus_annotated_feature")
mirna_columns <- biomaRt::listAttributes(mouse_dataset)
mirna_columns
##                        name              description              page
## 1                    efo_id       EFO Term Accession annotated_feature
## 2           chromosome_name          Chromosome Name annotated_feature
## 3          chromosome_start               Start (bp) annotated_feature
## 4            chromosome_end                 End (bp) annotated_feature
## 5         feature_type_name             Feature Type annotated_feature
## 6        feature_type_class       Feature Type class annotated_feature
## 7  feature_type_description Feature Type Description annotated_feature
## 8            epigenome_name           Epigenome name annotated_feature
## 9     epigenome_description    Epigenome Description annotated_feature
## 10             project_name             Project name annotated_feature
## 11               archive_id SRA Experiment Accession annotated_feature
## 12             so_accession        SO Term Accession annotated_feature
## 13                  so_name             SO Term Name annotated_feature
new_columns <- c("efo_id", "so_accession", "feature_type_name")
## 2016-09-28 This seems to be timing out right now.
## new_columns_download <- biomaRt::getBM(attributes=new_columns, mart=mouse_dataset)

## In contrast, when working with the transcripts from mice, we can pull
## everything directly from the OrganismDbi.
mm_tx_org <- sm(load_annotations(Mus.musculus, keytype="ensembltrans",
                                 fields=c("cdschrom", "definition", "genename")))
## I think the gene IDs returned by this are already perfect for the transcripts
## in the count tables.
mm_tx_genes <- mm_tx_org$genes
## The inclusion of the 'definition' columns means that we also end up with a bunch
## of redundant entries, like 200,000 of them; therefore I am going to whittle down
## the list.
mm_tx_unique <- !grepl(pattern="\\.", x=rownames(mm_tx_genes))
mm_tx_genes <- mm_tx_genes[mm_tx_unique, ]

The Mus.musculus data does not seem to include the full set of genes, lets compare to what we get when using biomart.

##mmtx_annotations <- get_biomart_annotations(species="mmusculus")
mart <- biomaRt::useMart(biomart="ENSEMBL_MART_ENSEMBL", host="dec2015.archive.ensembl.org")
dataset <- paste0("mmusculus_gene_ensembl")
ensembl <- try(biomaRt::useDataset(dataset, mart=mart))
lots_of_rows <- biomaRt::listAttributes(ensembl)  ## List of possible attributes
## wanted_attributes <- c("ensembl_gene_id", "ensembl_transcript_id", "ensembl_peptide_id", "chromosome_name", "start_position","end_position","description", "entrezgene","hgnc_symbol","hgnc_id","uniprot_sptrembl","uniprot_swissprot","uniprot_genename")  ## attributes Lucia and I chose, but too many
wanted_attributes_global <- c("ensembl_transcript_id", "ensembl_gene_id", "chromosome_name",
                              "start_position", "end_position", "strand", "description")
wanted_attributes_names <- c("ensembl_transcript_id", "entrezgene", "hgnc_symbol", "hgnc_id")
wanted_attributes_uniprot <- c("ensembl_transcript_id", "uniprot_sptrembl", "uniprot_swissprot")

wanted_global_annotations <- biomaRt::getBM(attributes=wanted_attributes_global, mart=ensembl)
dim(wanted_global_annotations)
## [1] 114083      7
wanted_names_annotations <- biomaRt::getBM(attributes=wanted_attributes_names, mart=ensembl)
wanted_uniprot_annotations <- biomaRt::getBM(attributes=wanted_attributes_uniprot, mart=ensembl)
dim(wanted_uniprot_annotations)
## [1] 57040     3
wanted_annotations <- merge(wanted_global_annotations, wanted_uniprot_annotations,
                            by.x="ensembl_transcript_id", by.y="ensembl_transcript_id", all.y=TRUE)

length(unique(wanted_annotations$ensembl_transcript_id))
## [1] 56999
rownames(wanted_annotations) <- make.names(wanted_annotations[["ensembl_transcript_id"]], unique=TRUE)

1.2 Prepare annotation tables

The annotation data given by load_annotations() is nice, but needs a little cleaning to match up with the count tables from the experiment.

## I happen to know that the gene IDs of the miRNA data are of the format
## 'chrx_ENSMUSG' so I will change the rownames of this slightly.
mm_mi_genes[["ID"]] <- paste0("chr", mm_mi_genes[["chromosome_name"]],
                              "_", mm_mi_genes[["ensembl_gene_id"]])
rownames(mm_mi_genes) <- make.names(mm_mi_genes[["ID"]], unique=TRUE)
rownames(mm_tx_genes) <- make.names(mm_tx_genes$ensembltrans, unique=TRUE)

2 Creating an Expressionset

The expressionset is the primary sequencing data format used in R. It brings together the experiment metadata (condition/batch/etc), gene annotations, and observed counts. The excel file containing the metadata also provides filenames containing the appropriate count tables.

mm_mi <- sm(create_expt(metadata="sample_sheets/all_samplesv1.xlsx", gene_info=mm_mi_genes,
                        file_column="mifile"))
## The IDs in the expressionset for the miRNAs are of the format 'chr1_ENSMUSG....'
mm_tx <- sm(create_expt(metadata="sample_sheets/all_samplesv1.xlsx", gene_info=wanted_annotations,
                        file_column="txfile"))
## The IDs in the expressionset for transcripts are of the format 'ENSMUST....'

In the previous block, we created the primary expressionsets used in the following work. However, much of it will need to performed with subsets of this data, so let us create those here.

## Because of the shenanigans I did to change the seed length for alignments, the counts are not
## guaranteed to be in integers, so round them here.
mm_mir <- mm_mi
Biobase::exprs(mm_mir$expressionset) <- round(Biobase::exprs(mm_mir$expressionset))
mm_txr <- mm_tx
Biobase::exprs(mm_txr$expressionset) <- round(Biobase::exprs(mm_txr$expressionset))

## Subsets!
mmmi_small <- expt_subset(mm_mir, subset="librarytype=='small'")
mmmi_large <- expt_subset(mm_mir, subset="librarytype=='large'")
mmtx_small <- expt_subset(mm_txr, subset="librarytype=='small'")
mmtx_large <- expt_subset(mm_txr, subset="librarytype=='large'")

mature_annot <- read.table("reference/mi_mappings.tab")
mature_fivep <- mature_annot[, c("fivep_accession","fivep_id")]
fivep_completed <- as.logical(mature_fivep[[1]] != "")
mature_fivep <- mature_fivep[fivep_completed, ]
mature_threep <- mature_annot[, c("threep_accession","threep_id")]
threep_completed <- as.logical(mature_threep[[1]] != "" & !is.na(mature_threep[[1]]))
mature_threep <- mature_threep[threep_completed, ]
colnames(mature_fivep) <- c("accession","id")
colnames(mature_threep) <- c("accession","id")
all_mature <- as.data.frame(rbind(mature_fivep, mature_threep))
rownames(all_mature) <-  make.names(gsub(pattern="\\-", replacement="\\.", x=all_mature$id), unique=TRUE)

mmmi_mature <- sm(create_expt(metadata="sample_sheets/all_samplesv1.xlsx", gene_info=all_mature,
                              file_column="maturefile"))

At this point, we should have everything necessary to perform the various analyses. So save the current data for reuse elsewhere.

index.html

LS0tCnRpdGxlOiAiQW5ub3RhdGlvbiBkYXRhIHVzZWQgZm9yIE0ubXVzY3VsdXMgc2FtcGxlcy4iCmF1dGhvcjogImF0YiBhYmVsZXdAZ21haWwuY29tIgpkYXRlOiAiYHIgU3lzLkRhdGUoKWAiCm91dHB1dDoKIGh0bWxfZG9jdW1lbnQ6CiAgY29kZV9kb3dubG9hZDogdHJ1ZQogIGNvZGVfZm9sZGluZzogc2hvdwogIGZpZ19jYXB0aW9uOiB0cnVlCiAgZmlnX2hlaWdodDogNwogIGZpZ193aWR0aDogNwogIGhpZ2hsaWdodDogdGFuZ28KICBrZWVwX21kOiBmYWxzZQogIG1vZGU6IHNlbGZjb250YWluZWQKICBudW1iZXJfc2VjdGlvbnM6IHRydWUKICBzZWxmX2NvbnRhaW5lZDogdHJ1ZQogIHRoZW1lOiBjb3NtbwogIHRvYzogdHJ1ZQogIHRvY19mbG9hdDoKICAgIGNvbGxhcHNlZDogZmFsc2UKICAgIHNtb290aF9zY3JvbGw6IGZhbHNlCi0tLQoKPHN0eWxlPgogIDwhLS0gRG9jdW1lbnQgcHJlbHVkZSByZXZpc2lvbnMgMjAxNi0xMCAtLT4KICBib2R5IC5tYWluLWNvbnRhaW5lciB7CiAgICBtYXgtd2lkdGg6IDE2MDBweDsKICB9Cjwvc3R5bGU+CgpgYGB7ciBvcHRpb25zLCBpbmNsdWRlPUZBTFNFfQojIyBUaGVzZSBhcmUgdGhlIG9wdGlvbnMgSSB0ZW5kIHRvIGZhdm9yCmxpYnJhcnkoImhwZ2x0b29scyIpCnR0IDwtIHNtKGRldnRvb2xzOjpsb2FkX2FsbCgifi9ocGdsdG9vbHMiKSkKa25pdHI6Om9wdHNfa25pdCRzZXQoCiAgICBwcm9ncmVzcyA9IFRSVUUsCiAgICB2ZXJib3NlID0gVFJVRSwKICAgIHdpZHRoID0gOTAsCiAgICBlY2hvID0gTlVMTCkKa25pdHI6Om9wdHNfY2h1bmskc2V0KAogICAgZXJyb3IgPSBUUlVFLAogICAgZmlnLndpZHRoID0gOCwKICAgIGZpZy5oZWlnaHQgPSA4LAogICAgZHBpID0gOTYpCm9sZF9vcHRpb25zIDwtIG9wdGlvbnMoCiAgICBlcnJvciA9IE5VTEwsCiAgICBkaWdpdHMgPSA0LAogICAgc3RyaW5nc0FzRmFjdG9ycyA9IEZBTFNFLAogICAga25pdHIuZHVwbGljYXRlLmxhYmVsID0gImFsbG93IikKZ2dwbG90Mjo6dGhlbWVfc2V0KGdncGxvdDI6OnRoZW1lX2J3KGJhc2Vfc2l6ZT0xMCkpCnNldC5zZWVkKDEpCnByZXZpb3VzX2ZpbGUgPC0gImluZGV4LlJtZCIKcm1kX2ZpbGUgPC0gImFubm90YXRpb24uUm1kIgp2ZXIgPC0gIjIwMTcwMzAxIgpwcmV2aW91c19zYXZlIDwtIHBhc3RlMChnc3ViKHBhdHRlcm49IlxcLlJtZCIsIHJlcGxhY2U9Ii5yZGEueHoiLCB4PXByZXZpb3VzX2ZpbGUpKQp0aGlzX3NhdmUgPC0gcGFzdGUwKGdzdWIocGF0dGVybj0iXFwuUm1kIiwgcmVwbGFjZT0iLnJkYS54eiIsIHg9cm1kX2ZpbGUpKQpgYGAKCltpbmRleC5odG1sXShpbmRleC5odG1sKSBbcHJlcHJvY2Vzc2luZy5odG1sXShwcmVwcm9jZXNzaW5nLmh0bWwpCgpgYGB7ciByZW5kZXJpbmcsIGluY2x1ZGU9RkFMU0UsIGV2YWw9RkFMU0V9CiMjIFRoaXMgYmxvY2sgaXMgdXNlZCB0byByZW5kZXIgYSBkb2N1bWVudCBmcm9tIHdpdGhpbiBpdC4Kcm1hcmtkb3duOjpyZW5kZXIocm1kX2ZpbGUpCgojIyBBbiBleHRyYSByZW5kZXJlciBmb3IgcGRmIG91dHB1dApybWFya2Rvd246OnJlbmRlcihybWRfZmlsZSwgb3V0cHV0X2Zvcm1hdD0icGRmX2RvY3VtZW50Iiwgb3V0cHV0X29wdGlvbnM9Yygic2tpcF9odG1sIikpCiMjIE9yIHRvIHNhdmUvbG9hZCBsYXJnZSBSZGF0YSBmaWxlcy4KaHBnbHRvb2xzOjo6c2F2ZW1lKCkKaHBnbHRvb2xzOjo6bG9hZG1lKCkKcm0obGlzdD1scygpKQpgYGAKCiMgR2Vub21lIGFubm90YXRpb24gaW5wdXQKClRoZXJlIGFyZSBhIGZldyBtZXRob2RzIG9mIGltcG9ydGluZyBhbm5vdGF0aW9uIGRhdGEgaW50byBSLiAgVGhlICduZXcnIG1ldGhvZCBpcyB0byB1c2UgdGhlIERCSQppbnRlcmZhY2VzIHByb3ZpZGVkIGJ5IGJpb2NvbmR1Y3Rvci4gIFRoZXNlIHdlcmUgYSBiaXQgbW9yZSBkaWZmaWN1bHQgZm9yIG1lIHRvIGxlYXJuLCBidXQgbm93IHRoYXQKSSBoYXZlIGEgZ2VuZXJhbCBpZGVhIG9mIGhvdyB0aGV5IHdvcmssIEkgZ3J1ZGdpbmdseSBhZG1pdCB0aGF0IHRoZXkgYXJlIGZhciBzdXBlcmlvci4KCiMjIE9yZ2FuaXNtRGJpL09yZ0RiL1R4RGIKClRoZSBtb3VzZSBPcmdhbmlzbURiaSBpbnRlcmZhY2UgaXMgY29udGFpbmVkIHRoZSBsaWJyYXJ5ICdNdXMubXVzY3VsdXMuJyAgSXQgYnJpbmdzIHRvZ2V0aGVyIHRoZQpPcmdEYiBhbmQgVHhEYiBwYWNrYWdlcyBmb3IgdGhlIG1vdXNlOiAnb3JnLk1tLmVnLmRiJyBhbmQgJ1R4RGIuTW11c2N1bHVzLlVDU0MubW0xMC5rbm93bkdlbmUnCnJlc3BlY3RpdmVseS4gIEkgYW0gZ29pbmcgdG8gbmVlZCB0byBmaWd1cmUgb3V0IGhvdyB0byBpbnRlZ3JhdGUgdGhlIG1pUk5BIGFubm90YXRpb25zIGludG8gdGhlbSwKYnV0IGZvciB0aGUgbW9tZW50IEkgd2lsbCB3b3JrIGp1c3Qgd2l0aCB0aGVtIGFuZCBob3BlZnVsbHkgZG8gdGhlbSBpbiBhIHdheSB3aGljaCBpcyBjbGVhbmVyIGFuZApjbGVhcmVyIHRoYW4gYmVmb3JlLgoKYGBge3IgbW11c2N1bHVzX29yZ2RifQojIyBvcmdEYiBwYWNrYWdlcyBhcmUgcmVzcG9uc2libGUgZm9yIHByb3ZpZGluZyBtYXBwaW5nIHRvIG90aGVyIGRhdGFiYXNlcywKIyMgbGlrZSBHTy9LRUdHL0VudHJlei9FbnNlbWJsL2V0Ywp0bXAgPC0gc20obGlicmFyeSgib3JnLk1tLmVnLmRiIikpCiMjIEluIGNvbnRyYXN0LCBUeERiIHBhY2thZ2VzIGFyZSByZXNwb25zaWJsZSBmb3IgcHJvdmlkaW5nIGluZm9ybWF0aW9uIGFib3V0CiMjIGVhY2ggdHJhbnNjcmlwdCwgdGh1cyBwcm92aWRpbmcgYSBsaW5rIGJldHdlZW4gdGhlIGFjdHVhbCBnZW5vbWUgc2VxdWVuY2UKIyMgYW5kIHRoZSB0cmFuc2NyaXB0cyBlbmNvZGVkIHdpdGhpbi4KdG1wIDwtIHNtKGxpYnJhcnkoIlR4RGIuTW11c2N1bHVzLlVDU0MubW0xMC5rbm93bkdlbmUiKSkKIyMgdGhlIE11cy5tdXNjdWx1cyBwYWNrYWdlIGFjdHVhbGx5IGxvYWRzIHRoZSBwcmV2aW91cyAyLCBidXQgSSB3YW50IHRvIGJlIGV4cGxpY2l0Lgp0bXAgPC0gc20obGlicmFyeSgiTXVzLm11c2N1bHVzIikpCgojIyBUaGUgZm9sbG93aW5nIHNlY3Rpb24gZ2F0aGVycyBhbm5vdGF0aW9uIGRhdGEgZnJvbSB0aGUgMjAxNSBhcmNoaXZlIG9mIGVuc2VtYmwuCiMjIEluIHRoaXMgY2FzZSwgYWxsIHRoZSBhbm5vdGF0aW9ucyBhcmUgZXhwbGljaXRseSB0aGUgbWlyYmFzZSBkYXRhCiMjIGFsb25nIHdpdGggYSBzdWJzZXQgb2Ygb3RoZXIgZW5zZW1ibCBmaWVsZHMuCmVuc2VtYmxfZ2VuZXMgPC0gYmlvbWFSdDo6dXNlRW5zZW1ibCgiZW5zZW1ibCIsIGhvc3Q9ImRlYzIwMTUuYXJjaGl2ZS5lbnNlbWJsLm9yZyIpCmdlbmVzX2RhdGFzZXRzIDwtIGJpb21hUnQ6Omxpc3REYXRhc2V0cyhlbnNlbWJsX2dlbmVzKQptb3VzZV9nZW5lcyA8LSBiaW9tYVJ0Ojp1c2VFbnNlbWJsKGJpb21hcnQ9ImVuc2VtYmwiLCBkYXRhc2V0PSJtbXVzY3VsdXNfZ2VuZV9lbnNlbWJsIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBob3N0PSJkZWMyMDE1LmFyY2hpdmUuZW5zZW1ibC5vcmciKQptaXJuYV9hdHRyaWJzIDwtIGMoImVuc2VtYmxfZ2VuZV9pZCIsICJlbnNlbWJsX3RyYW5zY3JpcHRfaWQiLCAiZGVzY3JpcHRpb24iLAogICAgICAgICAgICAgICAgICAiZXh0ZXJuYWxfZ2VuZV9uYW1lIiwgIm1pcmJhc2VfYWNjZXNzaW9uIiwgIm1pcmJhc2VfaWQiLAogICAgICAgICAgICAgICAgICAibWdpX3RyYW5zY3JpcHRfbmFtZSIsICJjaHJvbW9zb21lX25hbWUiKQptb3VzZV9taXJuYV9hdHRyaWJzIDwtIGJpb21hUnQ6Omxpc3RBdHRyaWJ1dGVzKG1hcnQ9bW91c2VfZ2VuZXMpCm1tX21pX2dlbmVzIDwtIGJpb21hUnQ6OmdldEJNKGF0dHJpYnV0ZXM9bWlybmFfYXR0cmlicywgZmlsdGVycz0iYmlvdHlwZSIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHZhbHVlcz0ibWlSTkEiLCBtYXJ0PW1vdXNlX2dlbmVzKQojIyBUaGUgZm9sbG93aW5nIGlzIHVzZWQgYnkgbWlSTkF0YWIuUm1kCm1tX21pX2dlbmVzJElEIDwtIHBhc3RlMCgiY2hyIiwgbW1fbWlfZ2VuZXMkY2hyb21vc29tZV9uYW1lLCAiXyIsIG1tX21pX2dlbmVzJGVuc2VtYmxfZ2VuZV9pZCkKcm93bmFtZXMobW1fbWlfZ2VuZXMpIDwtIG1ha2UubmFtZXMobW1fbWlfZ2VuZXMkSUQsIHVuaXF1ZT1UUlVFKQoKbGlicmFyeShtaXJiYXNlLmRiKQpjaG9zZW5fZGYgPC0gbW1fbWlfZ2VuZXNbLCBjKCJlbnNlbWJsX2dlbmVfaWQiLCJtaXJiYXNlX2lkIildCmNob3Nlbl9kZiRmaXZlcF9hY2Nlc3Npb24gPC0gIiIKY2hvc2VuX2RmJGZpdmVwX2lkIDwtICIiCmNob3Nlbl9kZiR0aHJlZXBfYWNjZXNzaW9uIDwtICIiCmNob3Nlbl9kZiR0aHJlZXBfaWQgPC0gIiIKbWlrZXkgPC0gIm1tdS1taXItMzQ0aC0xIgpuYW1lcyA8LSByb3duYW1lcyhjaG9zZW5fZGYpCmZvciAobWludW0gaW4gMTpsZW5ndGgoY2hvc2VuX2RmJG1pcmJhc2VfaWQpKSB7CiAgICBtaWtleSA8LSBjaG9zZW5fZGYkbWlyYmFzZV9pZFtbbWludW1dXQogICAgaWYgKG1pa2V5ID09ICIiKSB7CiAgICAgICAgbmV4dAogICAgfQogICAgbWlfbWFwcGluZ3MgPC0gdHJ5KGdldChtaWtleSwgbWlyYmFzZU1BVFVSRSksIHNpbGVudD1UUlVFKQogICAgaWYgKGNsYXNzKG1pX21hcHBpbmdzKSA9PSAidHJ5LWVycm9yIikgewogICAgICAgIG5leHQKICAgIH0KICAgIG1pX2FjY2Vzc2lvbnMgPC0gbWlfbWFwcGluZ3NAbWF0dXJlQWNjZXNzaW9uCiAgICBtaV9uYW1lcyA8LSBtaV9tYXBwaW5nc0BtYXR1cmVOYW1lCiAgICBuYW1lIDwtIG5hbWVzW1ttaW51bV1dCiAgICBjaG9zZW5fZGZbbmFtZSwgImZpdmVwX2FjY2Vzc2lvbiJdIDwtIG1pX2FjY2Vzc2lvbnNbMV0KICAgIGNob3Nlbl9kZltuYW1lLCAidGhyZWVwX2FjY2Vzc2lvbiJdIDwtIG1pX2FjY2Vzc2lvbnNbMl0KICAgIGNob3Nlbl9kZltuYW1lLCAiZml2ZXBfaWQiXSA8LSBtaV9uYW1lc1sxXQogICAgY2hvc2VuX2RmW25hbWUsICJ0aHJlZXBfaWQiXSA8LSBtaV9uYW1lc1syXQp9CndyaXRlLnRhYmxlKHg9Y2hvc2VuX2RmLCBmaWxlPSJyZWZlcmVuY2UvbWlfbWFwcGluZ3MudGFiIikKCmVuc2VtYmxfbmV3IDwtIGJpb21hUnQ6OnVzZUVuc2VtYmwoInJlZ3VsYXRpb24iKQpuZXdfZGF0YXNldHMgPC0gYmlvbWFSdDo6bGlzdERhdGFzZXRzKGVuc2VtYmxfbmV3KQptb3VzZV9kYXRhc2V0IDwtIGJpb21hUnQ6OnVzZUVuc2VtYmwoYmlvbWFydD0icmVndWxhdGlvbiIsIGRhdGFzZXQ9Im1tdXNjdWx1c19hbm5vdGF0ZWRfZmVhdHVyZSIpCm1pcm5hX2NvbHVtbnMgPC0gYmlvbWFSdDo6bGlzdEF0dHJpYnV0ZXMobW91c2VfZGF0YXNldCkKbWlybmFfY29sdW1ucwpuZXdfY29sdW1ucyA8LSBjKCJlZm9faWQiLCAic29fYWNjZXNzaW9uIiwgImZlYXR1cmVfdHlwZV9uYW1lIikKIyMgMjAxNi0wOS0yOCBUaGlzIHNlZW1zIHRvIGJlIHRpbWluZyBvdXQgcmlnaHQgbm93LgojIyBuZXdfY29sdW1uc19kb3dubG9hZCA8LSBiaW9tYVJ0OjpnZXRCTShhdHRyaWJ1dGVzPW5ld19jb2x1bW5zLCBtYXJ0PW1vdXNlX2RhdGFzZXQpCgojIyBJbiBjb250cmFzdCwgd2hlbiB3b3JraW5nIHdpdGggdGhlIHRyYW5zY3JpcHRzIGZyb20gbWljZSwgd2UgY2FuIHB1bGwKIyMgZXZlcnl0aGluZyBkaXJlY3RseSBmcm9tIHRoZSBPcmdhbmlzbURiaS4KbW1fdHhfb3JnIDwtIHNtKGxvYWRfYW5ub3RhdGlvbnMoTXVzLm11c2N1bHVzLCBrZXl0eXBlPSJlbnNlbWJsdHJhbnMiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBmaWVsZHM9YygiY2RzY2hyb20iLCAiZGVmaW5pdGlvbiIsICJnZW5lbmFtZSIpKSkKIyMgSSB0aGluayB0aGUgZ2VuZSBJRHMgcmV0dXJuZWQgYnkgdGhpcyBhcmUgYWxyZWFkeSBwZXJmZWN0IGZvciB0aGUgdHJhbnNjcmlwdHMKIyMgaW4gdGhlIGNvdW50IHRhYmxlcy4KbW1fdHhfZ2VuZXMgPC0gbW1fdHhfb3JnJGdlbmVzCiMjIFRoZSBpbmNsdXNpb24gb2YgdGhlICdkZWZpbml0aW9uJyBjb2x1bW5zIG1lYW5zIHRoYXQgd2UgYWxzbyBlbmQgdXAgd2l0aCBhIGJ1bmNoCiMjIG9mIHJlZHVuZGFudCBlbnRyaWVzLCBsaWtlIDIwMCwwMDAgb2YgdGhlbTsgdGhlcmVmb3JlIEkgYW0gZ29pbmcgdG8gd2hpdHRsZSBkb3duCiMjIHRoZSBsaXN0LgptbV90eF91bmlxdWUgPC0gIWdyZXBsKHBhdHRlcm49IlxcLiIsIHg9cm93bmFtZXMobW1fdHhfZ2VuZXMpKQptbV90eF9nZW5lcyA8LSBtbV90eF9nZW5lc1ttbV90eF91bmlxdWUsIF0KYGBgCgpUaGUgTXVzLm11c2N1bHVzIGRhdGEgZG9lcyBub3Qgc2VlbSB0byBpbmNsdWRlIHRoZSBmdWxsIHNldCBvZiBnZW5lcywgbGV0cyBjb21wYXJlIHRvIHdoYXQgd2UgZ2V0IHdoZW4gdXNpbmcgYmlvbWFydC4KCmBgYHtyIGNvbXBhcmVfdHhfYmlvbWFydH0KIyNtbXR4X2Fubm90YXRpb25zIDwtIGdldF9iaW9tYXJ0X2Fubm90YXRpb25zKHNwZWNpZXM9Im1tdXNjdWx1cyIpCm1hcnQgPC0gYmlvbWFSdDo6dXNlTWFydChiaW9tYXJ0PSJFTlNFTUJMX01BUlRfRU5TRU1CTCIsIGhvc3Q9ImRlYzIwMTUuYXJjaGl2ZS5lbnNlbWJsLm9yZyIpCmRhdGFzZXQgPC0gcGFzdGUwKCJtbXVzY3VsdXNfZ2VuZV9lbnNlbWJsIikKZW5zZW1ibCA8LSB0cnkoYmlvbWFSdDo6dXNlRGF0YXNldChkYXRhc2V0LCBtYXJ0PW1hcnQpKQpsb3RzX29mX3Jvd3MgPC0gYmlvbWFSdDo6bGlzdEF0dHJpYnV0ZXMoZW5zZW1ibCkgICMjIExpc3Qgb2YgcG9zc2libGUgYXR0cmlidXRlcwojIyB3YW50ZWRfYXR0cmlidXRlcyA8LSBjKCJlbnNlbWJsX2dlbmVfaWQiLCAiZW5zZW1ibF90cmFuc2NyaXB0X2lkIiwgImVuc2VtYmxfcGVwdGlkZV9pZCIsICJjaHJvbW9zb21lX25hbWUiLCAic3RhcnRfcG9zaXRpb24iLCJlbmRfcG9zaXRpb24iLCJkZXNjcmlwdGlvbiIsICJlbnRyZXpnZW5lIiwiaGduY19zeW1ib2wiLCJoZ25jX2lkIiwidW5pcHJvdF9zcHRyZW1ibCIsInVuaXByb3Rfc3dpc3Nwcm90IiwidW5pcHJvdF9nZW5lbmFtZSIpICAjIyBhdHRyaWJ1dGVzIEx1Y2lhIGFuZCBJIGNob3NlLCBidXQgdG9vIG1hbnkKd2FudGVkX2F0dHJpYnV0ZXNfZ2xvYmFsIDwtIGMoImVuc2VtYmxfdHJhbnNjcmlwdF9pZCIsICJlbnNlbWJsX2dlbmVfaWQiLCAiY2hyb21vc29tZV9uYW1lIiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgInN0YXJ0X3Bvc2l0aW9uIiwgImVuZF9wb3NpdGlvbiIsICJzdHJhbmQiLCAiZGVzY3JpcHRpb24iKQp3YW50ZWRfYXR0cmlidXRlc19uYW1lcyA8LSBjKCJlbnNlbWJsX3RyYW5zY3JpcHRfaWQiLCAiZW50cmV6Z2VuZSIsICJoZ25jX3N5bWJvbCIsICJoZ25jX2lkIikKd2FudGVkX2F0dHJpYnV0ZXNfdW5pcHJvdCA8LSBjKCJlbnNlbWJsX3RyYW5zY3JpcHRfaWQiLCAidW5pcHJvdF9zcHRyZW1ibCIsICJ1bmlwcm90X3N3aXNzcHJvdCIpCgp3YW50ZWRfZ2xvYmFsX2Fubm90YXRpb25zIDwtIGJpb21hUnQ6OmdldEJNKGF0dHJpYnV0ZXM9d2FudGVkX2F0dHJpYnV0ZXNfZ2xvYmFsLCBtYXJ0PWVuc2VtYmwpCmRpbSh3YW50ZWRfZ2xvYmFsX2Fubm90YXRpb25zKQp3YW50ZWRfbmFtZXNfYW5ub3RhdGlvbnMgPC0gYmlvbWFSdDo6Z2V0Qk0oYXR0cmlidXRlcz13YW50ZWRfYXR0cmlidXRlc19uYW1lcywgbWFydD1lbnNlbWJsKQp3YW50ZWRfdW5pcHJvdF9hbm5vdGF0aW9ucyA8LSBiaW9tYVJ0OjpnZXRCTShhdHRyaWJ1dGVzPXdhbnRlZF9hdHRyaWJ1dGVzX3VuaXByb3QsIG1hcnQ9ZW5zZW1ibCkKZGltKHdhbnRlZF91bmlwcm90X2Fubm90YXRpb25zKQp3YW50ZWRfYW5ub3RhdGlvbnMgPC0gbWVyZ2Uod2FudGVkX2dsb2JhbF9hbm5vdGF0aW9ucywgd2FudGVkX3VuaXByb3RfYW5ub3RhdGlvbnMsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICBieS54PSJlbnNlbWJsX3RyYW5zY3JpcHRfaWQiLCBieS55PSJlbnNlbWJsX3RyYW5zY3JpcHRfaWQiLCBhbGwueT1UUlVFKQoKbGVuZ3RoKHVuaXF1ZSh3YW50ZWRfYW5ub3RhdGlvbnMkZW5zZW1ibF90cmFuc2NyaXB0X2lkKSkKcm93bmFtZXMod2FudGVkX2Fubm90YXRpb25zKSA8LSBtYWtlLm5hbWVzKHdhbnRlZF9hbm5vdGF0aW9uc1tbImVuc2VtYmxfdHJhbnNjcmlwdF9pZCJdXSwgdW5pcXVlPVRSVUUpCmBgYAoKIyMgUHJlcGFyZSBhbm5vdGF0aW9uIHRhYmxlcwoKVGhlIGFubm90YXRpb24gZGF0YSBnaXZlbiBieSBsb2FkX2Fubm90YXRpb25zKCkgaXMgbmljZSwgYnV0IG5lZWRzIGEgbGl0dGxlIGNsZWFuaW5nIHRvIG1hdGNoIHVwCndpdGggdGhlIGNvdW50IHRhYmxlcyBmcm9tIHRoZSBleHBlcmltZW50LgoKYGBge3IgY2xlYW51cF9hbm5vdGF0aW9uc30KIyMgSSBoYXBwZW4gdG8ga25vdyB0aGF0IHRoZSBnZW5lIElEcyBvZiB0aGUgbWlSTkEgZGF0YSBhcmUgb2YgdGhlIGZvcm1hdAojIyAnY2hyeF9FTlNNVVNHJyBzbyBJIHdpbGwgY2hhbmdlIHRoZSByb3duYW1lcyBvZiB0aGlzIHNsaWdodGx5LgptbV9taV9nZW5lc1tbIklEIl1dIDwtIHBhc3RlMCgiY2hyIiwgbW1fbWlfZ2VuZXNbWyJjaHJvbW9zb21lX25hbWUiXV0sCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJfIiwgbW1fbWlfZ2VuZXNbWyJlbnNlbWJsX2dlbmVfaWQiXV0pCnJvd25hbWVzKG1tX21pX2dlbmVzKSA8LSBtYWtlLm5hbWVzKG1tX21pX2dlbmVzW1siSUQiXV0sIHVuaXF1ZT1UUlVFKQpyb3duYW1lcyhtbV90eF9nZW5lcykgPC0gbWFrZS5uYW1lcyhtbV90eF9nZW5lcyRlbnNlbWJsdHJhbnMsIHVuaXF1ZT1UUlVFKQpgYGAKCiMgQ3JlYXRpbmcgYW4gRXhwcmVzc2lvbnNldAoKVGhlIGV4cHJlc3Npb25zZXQgaXMgdGhlIHByaW1hcnkgc2VxdWVuY2luZyBkYXRhIGZvcm1hdCB1c2VkIGluIFIuICBJdCBicmluZ3MgdG9nZXRoZXIgdGhlCmV4cGVyaW1lbnQgbWV0YWRhdGEgKGNvbmRpdGlvbi9iYXRjaC9ldGMpLCBnZW5lIGFubm90YXRpb25zLCBhbmQgb2JzZXJ2ZWQgY291bnRzLiAgVGhlIGV4Y2VsIGZpbGUKY29udGFpbmluZyB0aGUgbWV0YWRhdGEgYWxzbyBwcm92aWRlcyBmaWxlbmFtZXMgY29udGFpbmluZyB0aGUgYXBwcm9wcmlhdGUgY291bnQgdGFibGVzLgoKYGBge3IgZXhwZXJpbWVudGFsX2Rlc2lnbn0KbW1fbWkgPC0gc20oY3JlYXRlX2V4cHQobWV0YWRhdGE9InNhbXBsZV9zaGVldHMvYWxsX3NhbXBsZXN2MS54bHN4IiwgZ2VuZV9pbmZvPW1tX21pX2dlbmVzLAogICAgICAgICAgICAgICAgICAgICAgICBmaWxlX2NvbHVtbj0ibWlmaWxlIikpCiMjIFRoZSBJRHMgaW4gdGhlIGV4cHJlc3Npb25zZXQgZm9yIHRoZSBtaVJOQXMgYXJlIG9mIHRoZSBmb3JtYXQgJ2NocjFfRU5TTVVTRy4uLi4nCm1tX3R4IDwtIHNtKGNyZWF0ZV9leHB0KG1ldGFkYXRhPSJzYW1wbGVfc2hlZXRzL2FsbF9zYW1wbGVzdjEueGxzeCIsIGdlbmVfaW5mbz13YW50ZWRfYW5ub3RhdGlvbnMsCiAgICAgICAgICAgICAgICAgICAgICAgIGZpbGVfY29sdW1uPSJ0eGZpbGUiKSkKIyMgVGhlIElEcyBpbiB0aGUgZXhwcmVzc2lvbnNldCBmb3IgdHJhbnNjcmlwdHMgYXJlIG9mIHRoZSBmb3JtYXQgJ0VOU01VU1QuLi4uJwpgYGAKCkluIHRoZSBwcmV2aW91cyBibG9jaywgd2UgY3JlYXRlZCB0aGUgcHJpbWFyeSBleHByZXNzaW9uc2V0cyB1c2VkIGluIHRoZSBmb2xsb3dpbmcgd29yay4gIEhvd2V2ZXIsIG11Y2ggb2YgaXQgd2lsbCBuZWVkIHRvIHBlcmZvcm1lZCB3aXRoIHN1YnNldHMgb2YgdGhpcyBkYXRhLCBzbyBsZXQgdXMgY3JlYXRlIHRob3NlIGhlcmUuCgpgYGB7ciBjcmVhdGVfc3Vic2V0c30KIyMgQmVjYXVzZSBvZiB0aGUgc2hlbmFuaWdhbnMgSSBkaWQgdG8gY2hhbmdlIHRoZSBzZWVkIGxlbmd0aCBmb3IgYWxpZ25tZW50cywgdGhlIGNvdW50cyBhcmUgbm90CiMjIGd1YXJhbnRlZWQgdG8gYmUgaW4gaW50ZWdlcnMsIHNvIHJvdW5kIHRoZW0gaGVyZS4KbW1fbWlyIDwtIG1tX21pCkJpb2Jhc2U6OmV4cHJzKG1tX21pciRleHByZXNzaW9uc2V0KSA8LSByb3VuZChCaW9iYXNlOjpleHBycyhtbV9taXIkZXhwcmVzc2lvbnNldCkpCm1tX3R4ciA8LSBtbV90eApCaW9iYXNlOjpleHBycyhtbV90eHIkZXhwcmVzc2lvbnNldCkgPC0gcm91bmQoQmlvYmFzZTo6ZXhwcnMobW1fdHhyJGV4cHJlc3Npb25zZXQpKQoKIyMgU3Vic2V0cyEKbW1taV9zbWFsbCA8LSBleHB0X3N1YnNldChtbV9taXIsIHN1YnNldD0ibGlicmFyeXR5cGU9PSdzbWFsbCciKQptbW1pX2xhcmdlIDwtIGV4cHRfc3Vic2V0KG1tX21pciwgc3Vic2V0PSJsaWJyYXJ5dHlwZT09J2xhcmdlJyIpCm1tdHhfc21hbGwgPC0gZXhwdF9zdWJzZXQobW1fdHhyLCBzdWJzZXQ9ImxpYnJhcnl0eXBlPT0nc21hbGwnIikKbW10eF9sYXJnZSA8LSBleHB0X3N1YnNldChtbV90eHIsIHN1YnNldD0ibGlicmFyeXR5cGU9PSdsYXJnZSciKQoKbWF0dXJlX2Fubm90IDwtIHJlYWQudGFibGUoInJlZmVyZW5jZS9taV9tYXBwaW5ncy50YWIiKQptYXR1cmVfZml2ZXAgPC0gbWF0dXJlX2Fubm90WywgYygiZml2ZXBfYWNjZXNzaW9uIiwiZml2ZXBfaWQiKV0KZml2ZXBfY29tcGxldGVkIDwtIGFzLmxvZ2ljYWwobWF0dXJlX2ZpdmVwW1sxXV0gIT0gIiIpCm1hdHVyZV9maXZlcCA8LSBtYXR1cmVfZml2ZXBbZml2ZXBfY29tcGxldGVkLCBdCm1hdHVyZV90aHJlZXAgPC0gbWF0dXJlX2Fubm90WywgYygidGhyZWVwX2FjY2Vzc2lvbiIsInRocmVlcF9pZCIpXQp0aHJlZXBfY29tcGxldGVkIDwtIGFzLmxvZ2ljYWwobWF0dXJlX3RocmVlcFtbMV1dICE9ICIiICYgIWlzLm5hKG1hdHVyZV90aHJlZXBbWzFdXSkpCm1hdHVyZV90aHJlZXAgPC0gbWF0dXJlX3RocmVlcFt0aHJlZXBfY29tcGxldGVkLCBdCmNvbG5hbWVzKG1hdHVyZV9maXZlcCkgPC0gYygiYWNjZXNzaW9uIiwiaWQiKQpjb2xuYW1lcyhtYXR1cmVfdGhyZWVwKSA8LSBjKCJhY2Nlc3Npb24iLCJpZCIpCmFsbF9tYXR1cmUgPC0gYXMuZGF0YS5mcmFtZShyYmluZChtYXR1cmVfZml2ZXAsIG1hdHVyZV90aHJlZXApKQpyb3duYW1lcyhhbGxfbWF0dXJlKSA8LSAgbWFrZS5uYW1lcyhnc3ViKHBhdHRlcm49IlxcLSIsIHJlcGxhY2VtZW50PSJcXC4iLCB4PWFsbF9tYXR1cmUkaWQpLCB1bmlxdWU9VFJVRSkKCm1tbWlfbWF0dXJlIDwtIHNtKGNyZWF0ZV9leHB0KG1ldGFkYXRhPSJzYW1wbGVfc2hlZXRzL2FsbF9zYW1wbGVzdjEueGxzeCIsIGdlbmVfaW5mbz1hbGxfbWF0dXJlLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICBmaWxlX2NvbHVtbj0ibWF0dXJlZmlsZSIpKQpgYGAKCkF0IHRoaXMgcG9pbnQsIHdlIHNob3VsZCBoYXZlIGV2ZXJ5dGhpbmcgbmVjZXNzYXJ5IHRvIHBlcmZvcm0gdGhlIHZhcmlvdXMgYW5hbHlzZXMuICBTbyBzYXZlIHRoZSBjdXJyZW50IGRhdGEgZm9yIHJldXNlIGVsc2V3aGVyZS4KCmBgYHtyIHNhdmVfYW5ub3RhdGlvbiwgaW5jbHVkZT1GQUxTRX0KdG1wIDwtIHNtKHNhdmVtZShmaWxlbmFtZT10aGlzX3NhdmUpKQpgYGAKCltpbmRleC5odG1sXShpbmRleC5odG1sKQoKIyBQcmludCBpbnRlcmVzdGluZyB0YWJsZXMKCkhlcmUgSSBwcmludCBzb21lIG9mIHRoZSBpbnRlcmVzdGluZyB0YWJsZXMgY3JlYXRlZCBhYm92ZSBhcyBleGFtcGxlcyBvZiB3aGF0IGhhcyBoYXBwZW5lZCB0aHVzIGZhcgp0byB0aGUgZGF0YS4KCiMjIE1tdXNjdWx1cyBkZXNpZ24KCmBgYHtyIGthYmxlX21tdXNjdWx1c30Ka25pdHI6OmthYmxlKGhlYWQobW1fbWlfZ2VuZXMpKQprbml0cjo6a2FibGUoaGVhZChtbV90eF9nZW5lcykpCmtuaXRyOjprYWJsZShtbV9taSRkZXNpZ24pCmBgYAo=