1. How To Run

How To Run

To run this notebook on AWS yourself, simply run this awc cli command or the equivalent. Note you will need to configure and specify your own security group, iam profile, and ssh key.

aws ec2 run-instances \
 --region null \
 --instance-initiated-shutdown-behavior terminate \
 --image-id null \
 --instance-type null \
 --count 1 \
 --security-groups sg-00000000000000000 \
 --iam-instance-profile Arn="IAM-ARN" \
 --key-name SSHKEY \
 --user-data "$(cat <<- EOF 
	#!/bin/bash
	sudo -H -u ec2-user /bin/bash << UIS
		export CP="";
		export EMAIL="your@email.com";
		cd ~/;
		for jar in charsets.jar deploy.jar access-bridge-64.jar cldrdata.jar dnsns.jar jaccess.jar jfxrt.jar localedata.jar nashorn.jar sunec.jar sunjce_provider.jar sunmscapi.jar sunpkcs11.jar zipfs.jar javaws.jar jce.jar jfr.jar jfxswt.jar jsse.jar management-agent.jar plugin.jar resources.jar rt.jar classes chill_2.12-0.9.5.jar kryo-shaded-4.0.2.jar xbean-asm7-shaded-4.15.jar chill-java-0.9.5.jar kryo-5.0.2.jar reflectasm-1.11.9.jar objenesis-3.1.jar minlog-1.3.1.jar logback-classic-1.3.0-alpha5.jar logback-core-1.3.0-alpha5.jar javax.mail-1.6.2.jar checker-framework-1.7.0.jar slf4j-api-2.0.0-alpha1.jar jcl-over-slf4j-2.0.0-alpha1.jar log4j-over-slf4j-2.0.0-alpha1.jar classes guava-30.0-jre.jar failureaccess-1.0.1.jar listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar checker-qual-3.8.0.jar error_prone_annotations-2.3.4.jar j2objc-annotations-1.3.jar jsr305-3.0.2.jar json-20201115.jar kryo-serializers-0.45.jar jackson-core-2.12.0.jar jackson-databind-2.12.0.jar jackson-annotations-2.12.0.jar classes commons-lang3-3.11.jar classes aws-java-sdk-ec2-1.11.918.jar jmespath-java-1.11.918.jar aws-java-sdk-s3-1.11.918.jar aws-java-sdk-kms-1.11.918.jar aws-java-sdk-core-1.11.918.jar commons-logging-1.1.3.jar httpclient-4.5.13.jar httpcore-4.4.13.jar ion-java-1.0.2.jar jackson-dataformat-cbor-2.6.7.jar joda-time-2.8.1.jar aws-java-sdk-iam-1.11.918.jar aws-java-sdk-ses-1.11.918.jar mail-1.5.0-b01.jar kryonet-2.22.0-RC1.jar jsonbeans-0.7.jar jsch-0.1.55.jar annotations-20.1.0.jar jnr-posix-3.1.4.jar jnr-ffi-2.2.1.jar jffi-1.3.1.jar jffi-1.3.1-native.jar asm-9.0.jar asm-commons-9.0.jar asm-analysis-9.0.jar asm-tree-9.0.jar asm-util-9.0.jar jnr-a64asm-1.0.0.jar jnr-x86asm-1.0.2.jar jnr-constants-0.10.1.jar classes flexmark-all-0.62.2.jar flexmark-0.62.2.jar flexmark-ext-abbreviation-0.62.2.jar flexmark-util-0.62.2.jar flexmark-ext-admonition-0.62.2.jar flexmark-ext-anchorlink-0.62.2.jar flexmark-ext-aside-0.62.2.jar flexmark-ext-attributes-0.62.2.jar flexmark-ext-autolink-0.62.2.jar autolink-0.6.0.jar flexmark-ext-definition-0.62.2.jar flexmark-ext-emoji-0.62.2.jar flexmark-ext-enumerated-reference-0.62.2.jar flexmark-ext-escaped-character-0.62.2.jar flexmark-ext-footnotes-0.62.2.jar flexmark-ext-gfm-issues-0.62.2.jar flexmark-ext-gfm-strikethrough-0.62.2.jar flexmark-ext-gfm-tasklist-0.62.2.jar flexmark-ext-gfm-users-0.62.2.jar flexmark-ext-gitlab-0.62.2.jar flexmark-ext-jekyll-front-matter-0.62.2.jar flexmark-ext-jekyll-tag-0.62.2.jar flexmark-ext-media-tags-0.62.2.jar flexmark-ext-macros-0.62.2.jar flexmark-ext-ins-0.62.2.jar flexmark-ext-xwiki-macros-0.62.2.jar flexmark-ext-superscript-0.62.2.jar flexmark-ext-tables-0.62.2.jar flexmark-ext-toc-0.62.2.jar flexmark-ext-typographic-0.62.2.jar flexmark-ext-wikilink-0.62.2.jar flexmark-ext-yaml-front-matter-0.62.2.jar flexmark-ext-youtube-embedded-0.62.2.jar flexmark-html2md-converter-0.62.2.jar jsoup-1.13.1.jar flexmark-jira-converter-0.62.2.jar flexmark-pdf-converter-0.62.2.jar openhtmltopdf-core-1.0.0.jar openhtmltopdf-pdfbox-1.0.0.jar pdfbox-2.0.16.jar fontbox-2.0.16.jar xmpbox-2.0.16.jar graphics2d-0.24.jar openhtmltopdf-rtl-support-1.0.0.jar icu4j-59.1.jar openhtmltopdf-jsoup-dom-converter-1.0.0.jar flexmark-profile-pegdown-0.62.2.jar flexmark-util-ast-0.62.2.jar flexmark-util-builder-0.62.2.jar flexmark-util-collection-0.62.2.jar flexmark-util-data-0.62.2.jar flexmark-util-dependency-0.62.2.jar flexmark-util-format-0.62.2.jar flexmark-util-html-0.62.2.jar flexmark-util-misc-0.62.2.jar flexmark-util-options-0.62.2.jar flexmark-util-sequence-0.62.2.jar flexmark-util-visitor-0.62.2.jar flexmark-youtrack-converter-0.62.2.jar org.eclipse.jgit-5.10.0.202012080955-r.jar JavaEWAH-1.1.7.jar classes arpack_combined_all-0.1.jar core-1.1.2.jar netlib-native_ref-osx-x86_64-1.1-natives.jar native_ref-java-1.1.jar jniloader-1.1.jar netlib-native_ref-linux-x86_64-1.1-natives.jar netlib-native_ref-linux-i686-1.1-natives.jar netlib-native_ref-win-x86_64-1.1-natives.jar netlib-native_ref-win-i686-1.1-natives.jar netlib-native_ref-linux-armhf-1.1-natives.jar netlib-native_system-osx-x86_64-1.1-natives.jar native_system-java-1.1.jar netlib-native_system-linux-x86_64-1.1-natives.jar netlib-native_system-linux-i686-1.1-natives.jar netlib-native_system-linux-armhf-1.1-natives.jar netlib-native_system-win-x86_64-1.1-natives.jar netlib-native_system-win-i686-1.1-natives.jar ejml-all-0.40.jar ejml-core-0.40.jar ejml-fdense-0.40.jar ejml-ddense-0.40.jar ejml-cdense-0.40.jar ejml-zdense-0.40.jar ejml-dsparse-0.40.jar ejml-simple-0.40.jar ejml-fsparse-0.40.jar classes hadoop-common-3.3.0.jar hadoop-shaded-protobuf_3_7-1.0.0.jar commons-cli-1.2.jar commons-collections-3.2.2.jar javax.activation-api-1.2.0.jar jetty-server-9.4.20.v20190813.jar jetty-http-9.4.20.v20190813.jar jetty-io-9.4.20.v20190813.jar jetty-util-9.4.20.v20190813.jar jetty-servlet-9.4.20.v20190813.jar jetty-security-9.4.20.v20190813.jar jetty-webapp-9.4.20.v20190813.jar jetty-xml-9.4.20.v20190813.jar jsp-api-2.1.jar jersey-servlet-1.19.jar jersey-json-1.19.jar jettison-1.1.jar jaxb-impl-2.2.3-1.jar jackson-jaxrs-1.9.2.jar jackson-xc-1.9.2.jar commons-beanutils-1.9.4.jar commons-configuration2-2.1.1.jar re2j-1.1.jar hadoop-auth-3.3.0.jar nimbus-jose-jwt-7.9.jar jcip-annotations-1.0-1.jar json-smart-2.3.jar accessors-smart-1.2.jar kerb-simplekdc-1.0.1.jar kerb-client-1.0.1.jar kerby-config-1.0.1.jar kerb-common-1.0.1.jar kerb-crypto-1.0.1.jar kerb-util-1.0.1.jar token-provider-1.0.1.jar kerb-admin-1.0.1.jar kerb-server-1.0.1.jar kerb-identity-1.0.1.jar kerby-xdr-1.0.1.jar curator-client-4.2.0.jar htrace-core4-4.1.0-incubating.jar kerb-core-1.0.1.jar kerby-pkix-1.0.1.jar kerby-asn1-1.0.1.jar kerby-util-1.0.1.jar stax2-api-3.1.4.jar woodstox-core-5.0.3.jar dnsjava-2.1.7.jar aws-java-sdk-codecommit-1.11.918.jar classes classes classes classes smile-plot-2.6.0.jar smile-core-2.6.0.jar smile-data-2.6.0.jar smile-math-2.6.0.jar smile-graph-2.6.0.jar swingx-1.6.1.jar filters-2.0.235.jar swing-worker-1.1.jar graphviz-java-0.18.0.jar viz.js-for-graphviz-java-2.1.3.jar svgSalamander-1.1.3.jar nashorn-promise-0.1.1.jar commons-exec-1.3.jar j2v8_win32_x86_64-4.6.0.jar classes org.eclipse.jdt.core-3.23.0.jar org.eclipse.core.resources-3.13.900.jar org.eclipse.core.expressions-3.7.0.jar org.eclipse.core.runtime-3.20.0.jar org.eclipse.osgi-3.16.100.jar org.eclipse.equinox.common-3.14.0.jar org.eclipse.core.jobs-3.10.1000.jar org.eclipse.equinox.registry-3.10.0.jar org.eclipse.equinox.preferences-3.8.100.jar org.eclipse.core.contenttype-3.7.800.jar org.eclipse.equinox.app-1.5.0.jar org.eclipse.core.filesystem-1.7.700.jar org.eclipse.text-3.10.400.jar org.eclipse.core.commands-3.9.800.jar maven-embedder-3.6.3.jar maven-settings-3.6.3.jar maven-settings-builder-3.6.3.jar plexus-interpolation-1.25.jar maven-core-3.6.3.jar maven-repository-metadata-3.6.3.jar maven-artifact-3.6.3.jar maven-resolver-provider-3.6.3.jar maven-resolver-impl-1.4.1.jar maven-resolver-spi-1.4.1.jar org.eclipse.sisu.inject-0.3.4.jar plexus-component-annotations-2.1.0.jar maven-plugin-api-3.6.3.jar maven-model-3.6.3.jar maven-model-builder-3.6.3.jar maven-builder-support-3.6.3.jar maven-resolver-api-1.4.1.jar maven-resolver-util-1.4.1.jar maven-shared-utils-3.2.1.jar guice-4.2.1-no_aop.jar javax.inject-1.jar jsr250-api-1.0.jar plexus-utils-3.2.1.jar plexus-classworlds-2.6.0.jar org.eclipse.sisu.plexus-0.3.4.jar cdi-api-1.0.jar plexus-sec-dispatcher-1.4.jar plexus-cipher-1.7.jar jblas-1.2.5.jar jcudnn-11.1.1.jar jcublas-11.1.1.jar jcublas-natives-11.1.1-windows-x86_64.jar jcudnn-natives-11.1.1-windows-x86_64.jar jcudnn-natives-11.1.1.jar jcuda-natives-11.1.1.jar classes classes classes proto-1.15.0.jar tensorflow-1.15.0.jar libtensorflow-1.15.0.jar libtensorflow_jni_gpu-1.15.0.jar protobuf-java-4.0.0-rc-2.jar jcusparse-11.1.1.jar jcuda-11.1.1.jar jcuda-natives-11.1.1-windows-x86_64.jar jcusolver-11.1.1.jar javacpp-1.5.4.jar hdf5-platform-1.10.4-1.4.4.jar hdf5-1.10.4-1.4.4-linux-x86.jar hdf5-1.10.4-1.4.4-linux-x86_64.jar hdf5-1.10.4-1.4.4-linux-ppc64le.jar hdf5-1.10.4-1.4.4-macosx-x86_64.jar hdf5-1.10.4-1.4.4-windows-x86.jar hdf5-1.10.4-1.4.4-windows-x86_64.jar hdf5-1.10.4-1.4.4.jar classes classes jackson-module-scala_2.12-2.12.0.jar scala-reflect-2.12.12.jar lz4-java-1.7.1.jar spark-hive_2.12-3.0.1.jar hive-common-2.3.7.jar jline-2.12.jar json-1.8.jar dropwizard-metrics-hadoop-metrics2-reporter-0.1.2.jar hive-exec-2.3.7-core.jar hive-vector-code-gen-2.3.7.jar velocity-1.5.jar antlr-runtime-3.5.2.jar ST4-4.0.4.jar stax-api-1.0.1.jar hive-metastore-2.3.7.jar javolution-5.5.1.jar bonecp-0.8.0.RELEASE.jar HikariCP-2.5.1.jar datanucleus-api-jdo-4.2.4.jar datanucleus-rdbms-4.1.19.jar commons-pool-1.5.4.jar commons-dbcp-1.4.jar jdo-api-3.0.1.jar jta-1.1.jar javax.jdo-3.2.0-m3.jar transaction-api-1.1.jar hive-serde-2.3.7.jar opencsv-2.3.jar hive-shims-2.3.7.jar hive-shims-common-2.3.7.jar hive-shims-0.23-2.3.7.jar hive-shims-scheduler-2.3.7.jar hive-llap-common-2.3.7.jar hive-llap-client-2.3.7.jar commons-httpclient-3.1.jar jackson-mapper-asl-1.9.13.jar commons-codec-1.10.jar jodd-core-3.5.2.jar datanucleus-core-4.1.17.jar libthrift-0.12.0.jar libfb303-0.9.3.jar derby-10.12.1.1.jar scala-library-2.12.12.jar scala-compiler-2.12.12.jar scala-xml_2.12-2.0.0-M3.jar gson-2.8.6.jar commons-io-2.8.0.jar spark-core_2.12-3.0.1.jar paranamer-2.8.jar avro-1.8.2.jar jackson-core-asl-1.9.13.jar commons-compress-1.20.jar xz-1.5.jar avro-mapred-1.8.2-hadoop2.jar avro-ipc-1.8.2.jar hadoop-client-3.3.0.jar hadoop-hdfs-client-3.3.0.jar okhttp-2.7.5.jar okio-1.6.0.jar hadoop-yarn-api-3.3.0.jar jaxb-api-2.2.11.jar hadoop-yarn-client-3.3.0.jar websocket-client-9.4.20.v20190813.jar jetty-client-9.4.20.v20190813.jar websocket-common-9.4.20.v20190813.jar websocket-api-9.4.20.v20190813.jar jline-3.9.0.jar hadoop-mapreduce-client-core-3.3.0.jar hadoop-yarn-common-3.3.0.jar jackson-module-jaxb-annotations-2.10.3.jar jakarta.xml.bind-api-2.3.2.jar jakarta.activation-api-1.2.1.jar jackson-jaxrs-json-provider-2.10.3.jar jackson-jaxrs-base-2.10.3.jar hadoop-mapreduce-client-jobclient-3.3.0.jar hadoop-mapreduce-client-common-3.3.0.jar hadoop-annotations-3.3.0.jar spark-launcher_2.12-3.0.1.jar spark-kvstore_2.12-3.0.1.jar leveldbjni-all-1.8.jar spark-network-common_2.12-3.0.1.jar spark-network-shuffle_2.12-3.0.1.jar spark-unsafe_2.12-3.0.1.jar activation-1.1.1.jar curator-recipes-2.7.1.jar curator-framework-2.7.1.jar zookeeper-3.4.14.jar audience-annotations-0.5.0.jar javax.servlet-api-3.1.0.jar commons-text-1.9.jar jul-to-slf4j-2.0.0-alpha1.jar compress-lzf-1.0.3.jar snappy-java-1.1.8.2.jar zstd-jni-1.4.4-3.jar RoaringBitmap-0.7.45.jar shims-0.7.45.jar commons-net-3.1.jar json4s-jackson_2.12-3.6.6.jar json4s-core_2.12-3.6.6.jar json4s-ast_2.12-3.6.6.jar json4s-scalap_2.12-3.6.6.jar jersey-client-2.30.jar jakarta.ws.rs-api-2.1.6.jar jakarta.inject-2.6.1.jar jersey-common-2.30.jar jakarta.annotation-api-1.3.5.jar osgi-resource-locator-1.0.3.jar jersey-server-2.30.jar jersey-media-jaxb-2.30.jar jakarta.validation-api-2.0.2.jar jersey-container-servlet-2.30.jar jersey-container-servlet-core-2.30.jar jersey-hk2-2.30.jar hk2-locator-2.6.1.jar aopalliance-repackaged-2.6.1.jar hk2-api-2.6.1.jar hk2-utils-2.6.1.jar javassist-3.25.0-GA.jar netty-all-4.1.47.Final.jar stream-2.9.6.jar metrics-core-4.1.1.jar metrics-jvm-4.1.1.jar metrics-json-4.1.1.jar metrics-graphite-4.1.1.jar metrics-jmx-4.1.1.jar ivy-2.4.0.jar oro-2.0.8.jar pyrolite-4.30.jar py4j-0.10.9.jar spark-tags_2.12-3.0.1.jar commons-crypto-1.0.0.jar unused-1.0.0.jar spark-sql_2.12-3.0.1.jar univocity-parsers-2.9.0.jar spark-sketch_2.12-3.0.1.jar spark-catalyst_2.12-3.0.1.jar scala-parser-combinators_2.12-1.1.2.jar janino-3.0.16.jar commons-compiler-3.0.16.jar antlr4-runtime-4.7.1.jar arrow-vector-0.15.1.jar arrow-format-0.15.1.jar arrow-memory-0.15.1.jar flatbuffers-java-1.9.0.jar orc-core-1.5.10.jar orc-shims-1.5.10.jar commons-lang-2.6.jar aircompressor-0.10.jar threeten-extra-1.5.0.jar orc-mapreduce-1.5.10.jar hive-storage-api-2.7.1.jar parquet-column-1.10.1.jar parquet-common-1.10.1.jar parquet-encoding-1.10.1.jar parquet-hadoop-1.10.1.jar parquet-format-2.4.0.jar parquet-jackson-1.10.1.jar jcudnn-natives-11.1.1-linux-x86_64.jar jcuda-natives-11.1.1-linux-x86_64.jar commons-math3-3.6.1.jar hadoop-aws-3.3.0.jar wildfly-openssl-1.0.7.Final.jar jcusparse-natives-11.1.1-windows-x86_64.jar jcusolver-natives-11.1.1-windows-x86_64.jar jcusparse-natives-11.1.1-linux-x86_64.jar jcusolver-natives-11.1.1-linux-x86_64.jar; 
		do 
		  export FILE="~/lib/\\\$jar"; 
		  aws s3 cp "s3://simiacryptus/lib/\\\$jar" \\\$FILE; 
		  export CP="\\\$CP:\\\$FILE"; 
		done
		nohup java -Dspark.driver.memory=32g -Dspark.master=local[16] -Dspark.app.name=local -Dtendril.sessionId=4479d29a-b482-46cc-9991-d19556849c95 -Dtendril.bucket=simiacryptus -Djetty.git.hash=e1bc35120a6617ee3df052294e433f3a25ce7097 -Dtendril.keyspace=lib/ -Dtendril.localcp=~/lib/ -cp \\\$CP com.simiacryptus.aws.S3TaskRunner &
	UIS
EOF
)"