Add api to fetch api usage data

aq-ikhwa-tech · aq-ikhwa-tech · commit d9983aa8cd33 · 2023-12-20T19:17:05.000+05:00
diff --git a/server/api-service/lowcoder-infra/src/main/java/org/lowcoder/infra/serverlog/ServerLog.java b/server/api-service/lowcoder-infra/src/main/java/org/lowcoder/infra/serverlog/ServerLog.java
@@ -15,15 +15,17 @@
 @Builder
 public class ServerLog {
     private String userId;
+    private String orgId;
     private String urlPath;
     private String httpMethod;
     private String requestBody;
     private Map<String, String> queryParameters;
     private long createTime;
 
     @JsonCreator
-    private ServerLog(String userId, String urlPath, String httpMethod, String requestBody, Map<String, String> queryParameters, long createTime) {
+    private ServerLog(String userId, String orgId, String urlPath, String httpMethod, String requestBody, Map<String, String> queryParameters, long createTime) {
         this.userId = userId;
+        this.orgId = orgId;
         this.urlPath = urlPath;
         this.createTime = createTime;
         this.httpMethod = httpMethod;
diff --git a/server/api-service/lowcoder-infra/src/main/java/org/lowcoder/infra/serverlog/ServerLogRepository.java b/server/api-service/lowcoder-infra/src/main/java/org/lowcoder/infra/serverlog/ServerLogRepository.java
@@ -1,8 +1,13 @@
 package org.lowcoder.infra.serverlog;
 
 import org.springframework.data.mongodb.repository.ReactiveMongoRepository;
+import reactor.core.publisher.Mono;
 
 public interface ServerLogRepository extends ReactiveMongoRepository<ServerLog, String> {
 
+    Mono<Long> countByOrgId(String orgId);
+
+    Mono<Long> countByOrgIdAndCreateTimeBetween(String orgId, Long startMonthEpoch, Long endMonthEpoch);
+
 }
 
diff --git a/server/api-service/lowcoder-infra/src/main/java/org/lowcoder/infra/serverlog/ServerLogService.java b/server/api-service/lowcoder-infra/src/main/java/org/lowcoder/infra/serverlog/ServerLogService.java
@@ -2,6 +2,9 @@
 
 import static org.lowcoder.infra.perf.PerfEvent.SERVER_LOG_BATCH_INSERT;
 
+import java.time.LocalDateTime;
+import java.time.ZoneOffset;
+import java.time.temporal.TemporalAdjusters;
 import java.util.Queue;
 import java.util.concurrent.ConcurrentLinkedQueue;
 import java.util.concurrent.TimeUnit;
@@ -13,6 +16,7 @@
 import org.springframework.stereotype.Service;
 
 import io.micrometer.core.instrument.Tags;
+import reactor.core.publisher.Mono;
 
 @Service
 public class ServerLogService {
@@ -42,4 +46,14 @@ private void scheduledInsert() {
                     perfHelper.count(SERVER_LOG_BATCH_INSERT, Tags.of("size", String.valueOf(result.size())));
                 });
     }
+
+    public Mono<Long> getApiUsageCount(String orgId, Boolean lastMonthOnly) {
+        if(lastMonthOnly != null && lastMonthOnly) {
+            Long startMonthEpoch = LocalDateTime.now().minusMonths(1).with(TemporalAdjusters.firstDayOfMonth()).toEpochSecond(ZoneOffset.UTC)*1000;
+            Long endMonthEpoch = LocalDateTime.now().minusMonths(1).with(TemporalAdjusters.lastDayOfMonth()).toEpochSecond(ZoneOffset.UTC)*1000;
+            return serverLogRepository.countByOrgIdAndCreateTimeBetween(orgId, startMonthEpoch, endMonthEpoch);
+        }
+        return serverLogRepository.countByOrgId(orgId);
+    }
+
 }
diff --git a/server/api-service/lowcoder-server/src/main/java/org/lowcoder/api/framework/filter/GlobalContextFilter.java b/server/api-service/lowcoder-server/src/main/java/org/lowcoder/api/framework/filter/GlobalContextFilter.java
@@ -71,18 +71,7 @@ public class GlobalContextFilter implements WebFilter, Ordered {
     public Mono<Void> filter(@Nonnull ServerWebExchange exchange, @Nonnull WebFilterChain chain) {
 
         return sessionUserService.getVisitorId()
-                .doOnNext(visitorId -> {
-                    if (isAnonymousUser(visitorId)) {
-                        return;
-                    }
-                    ServerLog serverLog = ServerLog.builder()
-                            .userId(visitorId)
-                            .urlPath(exchange.getRequest().getPath().toString())
-                            .httpMethod(Optional.ofNullable(exchange.getRequest().getMethod()).map(HttpMethod::name).orElse(""))
-                            .createTime(System.currentTimeMillis())
-                            .build();
-                    serverLogService.record(serverLog);
-                })
+                .flatMap(visitorId -> saveServerLog(exchange, visitorId))
                 .flatMap(visitorId -> chain.filter(exchange)
                         .contextWrite(ctx -> {
                             Map<String, Object> contextMap = buildContextMap(exchange, visitorId);
@@ -95,6 +84,27 @@ public Mono<Void> filter(@Nonnull ServerWebExchange exchange, @Nonnull WebFilter
                         }));
     }
 
+    private Mono<String> saveServerLog(ServerWebExchange exchange, String visitorId) {
+        if (isAnonymousUser(visitorId)) {
+            return Mono.just(visitorId);
+        }
+
+        return orgMemberService
+                .getCurrentOrgMember(visitorId)
+                .map(orgMember -> {
+                    ServerLog serverLog = ServerLog.builder()
+                            .orgId(orgMember.getOrgId())
+                            .userId(visitorId)
+                            .urlPath(exchange.getRequest().getPath().toString())
+                            .httpMethod(Optional.ofNullable(exchange.getRequest().getMethod()).map(HttpMethod::name).orElse(""))
+                            .createTime(System.currentTimeMillis())
+                            .build();
+                    serverLogService.record(serverLog);
+                    return visitorId;
+                });
+
+    }
+
     private Map<String, Object> buildContextMap(ServerWebExchange serverWebExchange, String visitorId) {
         ServerHttpRequest request = serverWebExchange.getRequest();
         Map<String, Object> contextMap = request.getHeaders().toSingleValueMap().entrySet()
diff --git a/server/api-service/lowcoder-server/src/main/java/org/lowcoder/api/usermanagement/OrgApiService.java b/server/api-service/lowcoder-server/src/main/java/org/lowcoder/api/usermanagement/OrgApiService.java
@@ -13,6 +13,7 @@
 
 import reactor.core.publisher.Mono;
 
+
 public interface OrgApiService {
 
     Mono<Boolean> leaveOrganization(String orgId);
@@ -45,5 +46,7 @@ public interface OrgApiService {
     Mono<Boolean> tryAddUserToOrgAndSwitchOrg(String orgId, String userId);
 
     Mono<ConfigView> getOrganizationConfigs(String orgId);
+
+    Mono<Long> getApiUsageCount(String orgId, Boolean lastMonthOnly);
 }
 
diff --git a/server/api-service/lowcoder-server/src/main/java/org/lowcoder/api/usermanagement/OrgApiServiceImpl.java b/server/api-service/lowcoder-server/src/main/java/org/lowcoder/api/usermanagement/OrgApiServiceImpl.java
@@ -38,6 +38,7 @@
 import org.lowcoder.domain.user.model.Connection;
 import org.lowcoder.domain.user.model.User;
 import org.lowcoder.domain.user.service.UserService;
+import org.lowcoder.infra.serverlog.ServerLogService;
 import org.lowcoder.sdk.auth.AbstractAuthConfig;
 import org.lowcoder.sdk.config.CommonConfig;
 import org.lowcoder.sdk.config.CommonConfig.Workspace;
@@ -76,6 +77,9 @@ public class OrgApiServiceImpl implements OrgApiService {
     @Autowired
     private AuthenticationService authenticationService;
 
+    @Autowired
+    private ServerLogService serverLogService;
+
     @Override
     public Mono<OrgMemberListView> getOrganizationMembers(String orgId, int page, int count) {
         return sessionUserService.getVisitorId()
@@ -373,6 +377,12 @@ public Mono<ConfigView> getOrganizationConfigs(String orgId) {
                 });
     }
 
+    @Override
+    public Mono<Long> getApiUsageCount(String orgId, Boolean lastMonthOnly) {
+        return checkVisitorAdminRole(orgId)
+                .flatMap(orgMember -> serverLogService.getApiUsageCount(orgId, lastMonthOnly));
+    }
+
     private Mono<Void> checkIfSaasMode() {
         return Mono.defer(() -> {
             if (commonConfig.getWorkspace().getMode() == WorkspaceMode.ENTERPRISE) {
diff --git a/server/api-service/lowcoder-server/src/main/java/org/lowcoder/api/usermanagement/OrganizationController.java b/server/api-service/lowcoder-server/src/main/java/org/lowcoder/api/usermanagement/OrganizationController.java
@@ -126,4 +126,10 @@ public Mono<ResponseView<Boolean>> updateOrgCommonSettings(@PathVariable String
                 .map(ResponseView::success);
     }
 
+    @Override
+    public Mono<ResponseView<Long>> getOrgApiUsageCount(String orgId, Boolean lastMonthOnly) {
+        return orgApiService.getApiUsageCount(orgId, lastMonthOnly)
+                .map(ResponseView::success);
+    }
+
 }
diff --git a/server/api-service/lowcoder-server/src/main/java/org/lowcoder/api/usermanagement/OrganizationEndpoints.java b/server/api-service/lowcoder-server/src/main/java/org/lowcoder/api/usermanagement/OrganizationEndpoints.java
@@ -160,6 +160,15 @@ public Mono<ResponseView<Boolean>> removeUserFromOrg(@PathVariable String orgId,
     @PutMapping("/{orgId}/common-settings")
     public Mono<ResponseView<Boolean>> updateOrgCommonSettings(@PathVariable String orgId, @RequestBody UpdateOrgCommonSettingsRequest request);
 
+	@Operation(
+			tags = TAG_ORGANIZATION_MANAGEMENT,
+			operationId = "getOrgApiUsageCount",
+			summary = "Get the api usage count for the org",
+			description = "Calculate the used api calls for this organization and return the count"
+	)
+	@GetMapping("/{orgId}/api-usage")
+	public Mono<ResponseView<Long>> getOrgApiUsageCount(@PathVariable String orgId, @RequestParam(required = false) Boolean lastMonthOnly);
+
     public record UpdateOrgCommonSettingsRequest(String key, Object value) {
 
     }
diff --git a/server/api-service/lowcoder-server/src/main/java/org/lowcoder/runner/migrations/DatabaseChangelog.java b/server/api-service/lowcoder-server/src/main/java/org/lowcoder/runner/migrations/DatabaseChangelog.java
@@ -175,6 +175,13 @@ public void completeAuthType(CompleteAuthType completeAuthType) {
         completeAuthType.complete();
     }
 
+    @ChangeSet(order = "019", id = "add-org-id-index-on-server-log", author = "")
+    public void addOrgIdIndexOnServerLog(MongockTemplate mongoTemplate) {
+        ensureIndexes(mongoTemplate, ServerLog.class,
+                makeIndex("orgId")
+        );
+    }
+
     public static Index makeIndex(String... fields) {
         if (fields.length == 1) {
             return new Index(fields[0], Sort.Direction.ASC).named(fields[0]);

Original file line number	Diff line number	Diff line change
`@@ -126,4 +126,10 @@ public Mono<ResponseView<Boolean>> updateOrgCommonSettings(@PathVariable String`
`126`	`126`	`.map(ResponseView::success);`
`127`	`127`	`}`
`128`	`128`
	`129`	`+ @Override`
	`130`	`+ public Mono<ResponseView<Long>> getOrgApiUsageCount(String orgId, Boolean lastMonthOnly) {`
	`131`	`+ return orgApiService.getApiUsageCount(orgId, lastMonthOnly)`
	`132`	`+ .map(ResponseView::success);`
	`133`	`+ }`
	`134`	`+`
`129`	`135`	`}`