Skip to content

Commit 81e0244

Browse files
parthshahparthshah
authored
Feature/search (#52)
* adding mock services for search * Feature/search (#51) * fix malformed json * add elasticsearch sync * implement elasticsearch indexing * update mock services and fetch member information * elasticsearch projects list api * update project lists tests * fix lint errors * fix minor errors * update postman * final fixes * update test configuration * some additional final fixes * fixing analytics and project history * adding support for aws-es, fixing bugs, refactoring * fixing path * fixing test script * updating test es endpoint * adding list endpoint along with es list * missing files * adding elasticsearch seed script
1 parent cdb2d1b commit 81e0244

39 files changed

+2044
-393
lines changed

‎.babelrc‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,9 @@
11
{
2-
"presets": ["es2015"]
2+
"presets": ["es2015"],
3+
"plugins": [
4+
["transform-runtime", {
5+
"polyfill": false,
6+
"regenerator": true
7+
}]
8+
]
39
}

‎README.md‎

Lines changed: 27 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
#Topcoder Projects Service
1+
# Topcoder Projects Service
22

33
Microservice to manage CRUD operations for all things Projects.
44

@@ -21,19 +21,42 @@ Copy config/sample.local.js as config/local.js, update the properties and accord
2121
Once you start your PostgreSQL database through docker, it will create a projectsDB.
2222
*To create tables - note this will drop tables if they already exist*
2323
```
24-
NODE_ENV=local npm run sync
24+
NODE_ENV=development npm run sync
2525
```
2626

2727
#### Redis
2828
Docker compose command will start a local redis instance as well. You should be able to connect to this instance using url `$(docker-machine ip):6379`
2929

3030
#### Elasticsearch
3131
Docker compose includes elasticsearch instance as well. It will open ports 9200 & 9300 (kibana)
32-
When creating the projects index, use the mapping provided in the local folder 'projects-es-mappings.json'
33-
`curl -XPUT --data @./local/projects-es-mappings.json http://dockerhost:9200/projects/`
32+
33+
#### Sync indices and mappings
34+
35+
There is a helper script to sync the indices and mappings with the elasticsearch.
36+
37+
Run `npm run elasticsearch:sync` from the root of project to execute the script.
38+
39+
> NOTE: This will first clear all the indices and than recreate them. So use with caution.
3440
3541
**NOTE**: In production these dependencies / services are hosted & managed outside tc-projects-service.
3642

43+
### Test
44+
45+
Each of the individual modules/services are unit tested.
46+
47+
To run unit tests run `npm run test` from root of project.
48+
49+
While tests are being executed the `NODE_ENV` environment variable has a value `test` and `config/test.js` configuration is loaded. The default test configuration refers to `projectsdb_test` postgres database. So make sure that this database exists before running the tests. Since we are using docker-compose for local deployment change `local/docker-compose.yaml` postgres service with updated database name and re-create the containers.
50+
51+
```
52+
// stop already executing containers if any
53+
docker-compose stop -t 1
54+
// clear the containers
55+
docker-compose rm -f
56+
// re-run the services with build flag
57+
docker-compose up --build
58+
```
59+
3760
#### JWT Authentication
3861
Authentication is handled via Authorization (Bearer) token header field. Token is a JWT token. Here is a sample token that is valid for a very long time for a user with administrator role.
3962
```

‎circle.yml‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
machine:
22

33
node:
4-
version: v5.7.0
4+
version: v6.9.4
55
environment:
66
DB_MASTER_URL: postgres://ubuntu:@127.0.0.1:5432/circle_test
77
#RABBITMQ_URL: amqp://localhost:5672
@@ -15,6 +15,12 @@ dependencies:
1515
- pip install awsebcli
1616
override:
1717
- npm install
18+
- wget https://download.elastic.co/elasticsearch/release/org/elasticsearch/distribution/tar/elasticsearch/2.3.5/elasticsearch-2.3.5.tar.gz
19+
- tar -xvf elasticsearch-2.3.5.tar.gz
20+
- elasticsearch-2.3.5/bin/elasticsearch: {background: true}
21+
# Make sure that Elasticsearch is up before running tests:
22+
- sleep 10 && wget --waitretry=5 --retry-connrefused -v http://127.0.0.1:9200/
23+
1824

1925
deployment:
2026
development:

‎config/custom-environment-variables.json‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,12 @@
55
"version": "APP_VERSION",
66
"captureLogs": "CAPTURE_LOGS",
77
"logentriesToken": "LOGENTRIES_TOKEN",
8-
"esUrl": "PROJECTS_ES_URL",
8+
"elasticsearchConfig": {
9+
"host": "PROJECTS_ES_URL",
10+
"apiVersion": "2.3",
11+
"indexName": "PROJECTS_ES_INDEX_NAME",
12+
"docType": "projectV4"
13+
},
914
"rabbitmqURL": "RABBITMQ_URL",
1015
"directProjectServiceEndpoint": "DIRECT_PROJECT_SERVICE_ENDPOINT",
1116
"directProjectServiceTimeout": "DIRECT_PROJECT_SERVICE_TIMEOUt",

‎config/default.json‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@
55
"captureLogs": "false",
66
"logentriesToken": "",
77
"rabbitmqURL": "",
8-
"esUrl": "",
98
"pubsubQueueName": "project.service",
109
"pubsubExchangeName": "projects",
1110
"fileServiceEndpoint": "",
@@ -16,6 +15,12 @@
1615
"attachmentsS3Bucket": "topcoder-prod-media",
1716
"projectAttachmentPathPrefix": "projects",
1817
"projectAttachmentPathSuffix": "attachments",
18+
"elasticsearchConfig": {
19+
"host": "",
20+
"apiVersion": "2.3",
21+
"indexName": "projects",
22+
"docType": "projectV4"
23+
},
1924
"systemUserClientId": "",
2025
"systemUserClientSecret": "",
2126
"userServiceUrl": "",

‎config/qa.json‎

Lines changed: 0 additions & 5 deletions
This file was deleted.

‎config/sample.local.js‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,8 @@ if (process.env.NODE_ENV === 'test') {
1313
fileServiceEndpoint: 'https://api.topcoder-dev.com/v3/files/',
1414
topicServiceEndpoint: 'https://api.topcoder-dev.com/v4/topics/',
1515
directProjectServiceEndpoint: 'https://api.topcoder-dev.com/v3/direct',
16-
userServiceUrl: 'https://api.topcoder-dev.com/v3/users',
1716
connectProjectsUrl: 'https://connect.topcoder-dev.com/projects/',
17+
memberServiceEndpoint: 'http://dockerhost:3001/members',
1818
salesforceLead: {
1919
webToLeadUrl: 'https://www.salesforce.com/servlet/servlet.WebToLead?encoding=UTF-8',
2020
orgId: '00D2C0000000dO6',
@@ -29,6 +29,13 @@ if (process.env.NODE_ENV === 'test') {
2929
minPoolSize: 4,
3030
idleTimeout: 1000,
3131
},
32+
elasticsearchConfig: {
33+
host: 'dockerhost:9200',
34+
// target elasticsearch 2.3 version
35+
apiVersion: '2.3',
36+
indexName: 'projects',
37+
docType: 'projectV4'
38+
},
3239
};
3340
}
3441
module.exports = config;

‎config/test.json‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,12 @@
33
"logLevel": "debug",
44
"captureLogs": "false",
55
"logentriesToken": "",
6-
"esUrl": "//localhost:9200",
6+
"elasticsearchConfig": {
7+
"host": "http://localhost:9200",
8+
"apiVersion": "2.3",
9+
"indexName": "projects_test",
10+
"docType": "projectV4"
11+
},
712
"rabbitmqUrl": "amqp://localhost:5672",
813
"dbConfig": {
914
"masterUrl": "postgres://coder:mysecretpassword@localhost:5432/projectsdb_test",

‎local/docker-compose.yml‎

Lines changed: 25 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,25 @@
1-
postgres:
2-
image: "postgres:9.5"
3-
ports:
4-
- "5432:5432"
5-
environment:
6-
- POSTGRES_PASSWORD=mysecretpassword
7-
- POSTGRES_USER=coder
8-
- POSTGRES_DB=projectsdb
9-
#elasticsearch:
10-
# image: "elasticsearch:2"
11-
# ports:
12-
# - "9200:9200"
13-
# - "9300:9300"
14-
redis:
15-
image: redis:2.8
16-
ports:
17-
- "6379:6379"
18-
rabbitmq:
19-
image: rabbitmq:3
20-
restart: always
21-
ports:
22-
- 5672:5672
1+
version: "2"
2+
services:
3+
jsonserver:
4+
build: "mock-services"
5+
ports:
6+
- "3001:3001"
7+
db:
8+
image: "postgres:9.5"
9+
ports:
10+
- "5432:5432"
11+
environment:
12+
- POSTGRES_PASSWORD=mysecretpassword
13+
- POSTGRES_USER=coder
14+
- POSTGRES_DB=projectsdb
15+
esearch:
16+
image: "elasticsearch:2.3"
17+
ports:
18+
- "9200:9200"
19+
- "9300:9300"
20+
queue:
21+
image: "rabbitmq:3-management"
22+
restart: always
23+
ports:
24+
- "5672:5672"
25+
- "15672:15672"

‎local/mock-services/.eslintrc‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
{
2+
"extends": "airbnb-base",
3+
"env": {
4+
"browser": false,
5+
"node": true,
6+
"es6": true,
7+
"mocha": true
8+
},
9+
"rules": {
10+
"import/no-unresolved": 0,
11+
"no-param-reassign": 0,
12+
"max-len": ["error", { "ignoreComments": true, "code": 120 }],
13+
"valid-jsdoc": ["error", {
14+
"requireReturn": true,
15+
"requireReturnType": true,
16+
"requireParamDescription": true,
17+
"requireReturnDescription": true
18+
}],
19+
"require-jsdoc": ["error", {
20+
"require": {
21+
"FunctionDeclaration": true,
22+
"MethodDefinition": true,
23+
"ClassDeclaration": true
24+
}
25+
}]
26+
}
27+
}

0 commit comments

Comments
 (0)