diff --git a/osstracker-console/Dockerfile b/osstracker-console/Dockerfile new file mode 100644 index 0000000..e8eed84 --- /dev/null +++ b/osstracker-console/Dockerfile @@ -0,0 +1,2 @@ +FROM node:onbuild +EXPOSE 3000 \ No newline at end of file diff --git a/osstracker-ddl/osstracker-simple.cql b/osstracker-ddl/osstracker-simple.cql new file mode 100644 index 0000000..33e760a --- /dev/null +++ b/osstracker-ddl/osstracker-simple.cql @@ -0,0 +1,25 @@ +DROP KEYSPACE IF EXISTS osstracker; + +CREATE KEYSPACE osstracker WITH replication = { + 'class': 'SimpleStrategy', + 'replication_factor': 3 +}; + +USE osstracker; + +CREATE TABLE repo_orgs ( + org_short text PRIMARY KEY, + org_description text +); + +CREATE TABLE repo_info ( + gh_repo_name text PRIMARY KEY, + mgr_lead_empid text, + gh_exists boolean, + gh_org text, + dev_lead_empid text, + org_short text, + gh_public boolean, + osslifecycle text, + last_stats_update timestamp +); diff --git a/osstracker-ddl/osstracker.cql b/osstracker-ddl/osstracker.cql index 52a4cf0..ec764d1 100644 --- a/osstracker-ddl/osstracker.cql +++ b/osstracker-ddl/osstracker.cql @@ -1,4 +1,4 @@ -DROP KEYSPACE osstracker; +DROP KEYSPACE IF EXISTS osstracker ; CREATE KEYSPACE osstracker WITH replication = { 'class': 'NetworkTopologyStrategy', diff --git a/osstracker-docs/HighLevelView.png b/osstracker-docs/HighLevelView.png new file mode 100644 index 0000000..0ea4c5b Binary files /dev/null and b/osstracker-docs/HighLevelView.png differ diff --git a/osstracker-docs/README.md b/osstracker-docs/README.md new file mode 100644 index 0000000..0e7786f --- /dev/null +++ b/osstracker-docs/README.md @@ -0,0 +1,93 @@ +Overview +======== + +You can see more about OSS Tracker from our meetup [video](https://www.youtube.com/watch?v=5s-SS_aXoi0) and [slides](http://www.slideshare.net/aspyker/netflix-open-source-meetup-season-4-episode-1). + +High Level View +=============== + +![High Level View](https://raw.githubusercontent.com/Netflix/osstracker/documentation/osstracker-docs/HighLevelView.png) + +Manual installation +=================== + +Following manual installation is sample installation based on Linux system using docker containers. You can adapt it easily for other system using *docker-machine*, but care about `CASS_HOST` and `ES_HOST` they should be equals to `docker-machine ip MACHINE_NAME`. + +**ATTENTION** neither *Cassandra* nor *ElasticSearch* containers define mounting volumes, if containers are removed data will be lost. + +Clone OSS Tracker source code + +``` +git clone https://github.com/Netflix/osstracker.git +``` + +Run Cassandra instance (using docker) + +``` +# At start export CASS_HOST that should represent your docker ip +# linux only +export CASS_HOST=localhost +# If you're using docker-machine (Windows/Mac OSX) you should replace by following commented command +# MacOSX / Windows (with bash) +# export CASS_HOST=$(docker-machine ip ) +export CASS_PORT=9042 +docker run -d --name osstracker-cassandra -p ${CASS_PORT}:9042 cassandra +``` + +Init Cassandra DDL + +``` +docker run -it --link osstracker-cassandra:cassandra -v $(pwd)/osstracker-ddl/:/tmp/ddl/ --rm cassandra cqlsh cassandra -f /tmp/ddl/osstracker-simple.cql +``` + +Run ElasticSearch instance (using docker) + +``` +# At start export ES_HOST that should represent your docker ip +# linux only +export ES_HOST=localhost +# If you're using docker-machine (Windows/Mac OSX) you should replace by following commented command +# MacOSX / Windows (with bash) +# export ES_HOST=$(docker-machine ip ) +export ES_PORT=9200 +docker run -d --name osstracker-elasticsearch -p ${ES_PORT}:9200 elasticsearch +``` + +Init ElasticSearch DDL + +``` +curl -i -X PUT ${ES_HOST}:${ES_PORT}/osstracker +curl -i -X PUT -H "Content-Type: application/json" -d '{"properties":{"repo_name":{"type":"string","index":"not_analyzed"}}}' ${ES_HOST}:${ES_PORT}/osstracker/_mapping/repo_stats +``` + +Populate Cassandra data and ElasticSearch data (order is important Cassandra first then ElasticSearch) + +``` +export GITHUB_KEY=XXXXXX +./gradlew -PCASS_HOST=${CASS_HOST} -PCASS_PORT=${CASS_PORT} -PES_HOST=${ES_HOST} -PES_PORT=${ES_PORT} -PGITHUB_OAUTH=${GITHUB_KEY} -PACTION=updatecassandra clean run +./gradlew -PCASS_HOST=${CASS_HOST} -PCASS_PORT=${CASS_PORT} -PES_HOST=${ES_HOST} -PES_PORT=${ES_PORT} -PGITHUB_OAUTH=${GITHUB_KEY} -PACTION=updateelasticsearch run +``` + +Run Kibana instance (using docker) +**ATTENTION** code source hardcodes *Kibana* host and port so *Kibana* host must be equals to *ElasticSearch* host + +``` +docker run -d --name osstracker-kibana --link osstracker-elasticsearch:elasticsearch -p 5601:5601 kibana +``` + +Create Docker OSS Tracker console image + +``` +cd osstracker-console/ +docker build -t netflixoss/osstracker-console . +``` + +Run OSS Tracker console (using docker) + +``` +docker run -d --name osstracker-console -e CASS_HOST=${CASS_HOST} -e CASS_PORT=${CASS_PORT} -e ES_HOST=${ES_HOST} -e ES_PORT=${ES_PORT} -p 3000:3000 netflixoss/osstracker-console +``` + +Enjoy + + diff --git a/osstracker-scraper/build.gradle b/osstracker-scraper/build.gradle index 13f7c2a..e17b956 100644 --- a/osstracker-scraper/build.gradle +++ b/osstracker-scraper/build.gradle @@ -22,13 +22,13 @@ task runScraper(type: JavaExec, dependsOn: classes) { environment = [ // You should update with your github OAUTH token - "github_oauth" : "1111111111111111111111111111111111111111", + "github_oauth" : project.hasProperty("GITHUB_OAUTH") ? GITHUB_OAUTH : "1111111111111111111111111111111111111111", // You should update with a a cassandra host and port "CASS_HOST" : CASS_HOST, - "CASS_PORT" : "7104", + "CASS_PORT" : project.hasProperty("CASS_PORT") ? CASS_PORT : "7104", // You should update with an elasticsearch host and port "ES_HOST" : ES_HOST, - "ES_PORT" : "7104" + "ES_PORT" : project.hasProperty("ES_PORT") ? ES_PORT : "7104" ] jvmArgs = [ @@ -37,6 +37,6 @@ task runScraper(type: JavaExec, dependsOn: classes) { args = [ //"--action", "updatecassandra", - "--action", "updateelasticsearch", + "--action", project.hasProperty("ACTION") ? ACTION : "updateelasticsearch", ] } \ No newline at end of file