[#12195] Add new agent list using uid

collector/src/main/java/com/navercorp/pinpoint/collector/handler/grpc/GrpcAgentInfoHandler.java

@@ -91,7 +91,7 @@
         try {
             // agent info
             final AgentInfoBo agentInfoBo = this.agentInfoBoMapper.map(agentInfo, header);
-            this.agentInfoService.insert(agentInfoBo);
+            this.agentInfoService.insert(header.getServiceUid(), header.getApplicationUid(), agentInfoBo);
             return PResult.newBuilder().setSuccess(true).build();
         } catch (Exception e) {
             logger.warn("Failed to handle. agentInfo={}", MessageFormatUtils.debugLog(agentInfo), e);

collector/src/main/java/com/navercorp/pinpoint/collector/service/AgentInfoService.java

@@ -18,14 +18,24 @@
 
 import com.navercorp.pinpoint.collector.dao.AgentInfoDao;
 import com.navercorp.pinpoint.collector.dao.ApplicationIndexDao;
+import com.navercorp.pinpoint.collector.uid.dao.AgentNameDao;
+import com.navercorp.pinpoint.collector.uid.service.ApplicationUidService;
 import com.navercorp.pinpoint.common.server.bo.AgentInfoBo;
+import com.navercorp.pinpoint.common.server.uid.ApplicationUid;
+import com.navercorp.pinpoint.common.server.uid.ServiceUid;
+import com.navercorp.pinpoint.io.request.UidException;
 import jakarta.validation.Valid;
 import jakarta.validation.constraints.NotBlank;
 import jakarta.validation.constraints.PositiveOrZero;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
 import org.springframework.validation.annotation.Validated;
 
 import java.util.Objects;
+import java.util.Optional;
+import java.util.function.Supplier;
 
 /**
  * @author emeroad
@@ -35,19 +45,59 @@
 @Service
 @Validated
 public class AgentInfoService {
+    private final Logger logger = LogManager.getLogger(this.getClass());
 
     private final AgentInfoDao agentInfoDao;
-
     private final ApplicationIndexDao applicationIndexDao;
 
-    public AgentInfoService(AgentInfoDao agentInfoDao, ApplicationIndexDao applicationIndexDao) {
+    private final ApplicationUidService applicationUidService;
+    private final AgentNameDao agentNameDao;
+
+    private final boolean uidAgentListEnable;
+
+    public AgentInfoService(AgentInfoDao agentInfoDao, ApplicationIndexDao applicationIndexDao,
+                            ApplicationUidService applicationUidService, Optional<AgentNameDao> agentNameDao,
+                            @Value("${pinpoint.collector.application.uid.agent.list.enable:false}") boolean uidAgentListEnable) {
         this.agentInfoDao = Objects.requireNonNull(agentInfoDao, "agentInfoDao");
         this.applicationIndexDao = Objects.requireNonNull(applicationIndexDao, "applicationIndexDao");
+        this.applicationUidService = Objects.requireNonNull(applicationUidService, "applicationUidService");
+        this.agentNameDao = agentNameDao.orElse(null);
+        this.uidAgentListEnable = uidAgentListEnable;
     }
 
-    public void insert(@Valid final AgentInfoBo agentInfoBo) {
+    public void insert(Supplier<ServiceUid> serviceUidSupplier, Supplier<ApplicationUid> applicationUidSupplier, @Valid final AgentInfoBo agentInfoBo) {
         agentInfoDao.insert(agentInfoBo);
         applicationIndexDao.insert(agentInfoBo);
+        uidAgentListInsert(serviceUidSupplier, applicationUidSupplier, agentInfoBo);
+    }
+
+    private void uidAgentListInsert(Supplier<ServiceUid> serviceUidSupplier, Supplier<ApplicationUid> applicationUidSupplier, AgentInfoBo agentInfoBo) {
+        if (!uidAgentListEnable || agentNameDao == null) {
+            return;
+        }
+        ServiceUid serviceUid = serviceUidSupplier.get();
+        ApplicationUid applicationUid = getApplicationUid(applicationUidSupplier, agentInfoBo, serviceUid);
+        insertApplicationName(agentInfoBo, serviceUid, applicationUid);
+    }
+
+    private ApplicationUid getApplicationUid(Supplier<ApplicationUid> applicationUidSupplier, AgentInfoBo agentInfoBo, ServiceUid serviceUid) {
+        try {
+            return applicationUidSupplier.get();
+        } catch (UidException exception) {
+            return applicationUidService.getOrCreateApplicationUid(serviceUid, agentInfoBo.getApplicationName());
+        }
+    }
+
+    private void insertApplicationName(AgentInfoBo agentInfoBo, ServiceUid serviceUid, ApplicationUid applicationUid) {
+        if (applicationUid == null) {
+            logger.warn("null applicationUid. {}, applicationName: {}, id: {}, name: {}", serviceUid, agentInfoBo.getApplicationName(), agentInfoBo.getAgentId(), agentInfoBo.getAgentName());
+            return;
+        }
+        try {
+            agentNameDao.insert(serviceUid, applicationUid, agentInfoBo.getAgentId(), agentInfoBo.getStartTime(), agentInfoBo.getAgentName());
+        } catch (Exception e) {
+            logger.warn("Failed to insert uid agent list. {}, applicationName: {}, id: {}, name: {}", serviceUid, agentInfoBo.getApplicationName(), agentInfoBo.getAgentId(), agentInfoBo.getAgentName(), e);
+        }
     }
 
     public AgentInfoBo getSimpleAgentInfo(@NotBlank final String agentId, @PositiveOrZero final long timestamp) {

collector/src/main/java/com/navercorp/pinpoint/collector/uid/UidModule.java

@@ -3,15 +3,8 @@
 import com.navercorp.pinpoint.collector.uid.config.ApplicationUidConfig;
 import com.navercorp.pinpoint.collector.uid.config.ServiceUidCacheConfig;
 import com.navercorp.pinpoint.collector.uid.config.UidHbaseTemplateConfiguration;
-import com.navercorp.pinpoint.collector.uid.service.ApplicationUidService;
-import com.navercorp.pinpoint.collector.uid.service.EmptyApplicationUidService;
-import com.navercorp.pinpoint.collector.uid.service.ServiceGroupService;
-import com.navercorp.pinpoint.collector.uid.service.StaticServiceGroupService;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
-import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.ComponentScan;
 import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.Import;
@@ -35,15 +28,4 @@ public UidModule() {
         logger.info("Install UidModule");
     }
 
-    @Bean
-    @ConditionalOnMissingBean(ApplicationUidService.class)
-    public ApplicationUidService emptyApplicationUidService() {
-        return new EmptyApplicationUidService();
-    }
-
-    @Bean
-    @ConditionalOnMissingBean(ServiceGroupService.class)
-    public ServiceGroupService defaultServiceGroupService(@Value("${collector.service.uid.default.value:0}") int staticServiceUid) {
-        return new StaticServiceGroupService(staticServiceUid);
-    }
 }

collector/src/main/java/com/navercorp/pinpoint/collector/uid/config/ApplicationUidConfig.java

@@ -1,11 +1,14 @@
 package com.navercorp.pinpoint.collector.uid.config;
 
 
+import com.github.benmanes.caffeine.cache.Cache;
 import com.github.benmanes.caffeine.cache.Caffeine;
 import com.navercorp.pinpoint.common.server.uid.ApplicationUid;
-import com.navercorp.pinpoint.common.server.uid.cache.CaffeineCacheSpec;
+import com.navercorp.pinpoint.common.server.uid.cache.CaffeineCacheProperties;
 import com.navercorp.pinpoint.common.server.util.IdGenerator;
 import com.navercorp.pinpoint.common.server.util.RandomApplicationUidGenerator;
+import io.micrometer.core.instrument.binder.MeterBinder;
+import io.micrometer.core.instrument.binder.cache.CaffeineCacheMetrics;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.springframework.beans.factory.annotation.Qualifier;
@@ -32,23 +35,58 @@
 
     @Bean
     @ConfigurationProperties(prefix = "collector.application.uid.cache")
-    public CaffeineCacheSpec applicationUidCacheSpec() {
-        return new CaffeineCacheSpec();
+    public CaffeineCacheProperties applicationUidCacheProperties() {
+        return new CaffeineCacheProperties();
     }
 
     @Bean
-    public CacheManager applicationUidCache(@Qualifier("applicationUidCacheSpec") CaffeineCacheSpec caffeineCacheSpec) {
+    public CacheManager applicationUidCache(@Qualifier("applicationUidCacheProperties") CaffeineCacheProperties caffeineCacheProperties) {
         CaffeineCacheManager cacheManager = new CaffeineCacheManager(APPLICATION_UID_CACHE_NAME);
-        cacheManager.setCaffeine(Caffeine.from(caffeineCacheSpec.getSpecification()));
+        cacheManager.setCaffeine(buildCaffeine(caffeineCacheProperties));
         cacheManager.setAsyncCacheMode(true);
         cacheManager.setAllowNullValues(false);
-
         return cacheManager;
     }
 
+    private Caffeine<Object, Object> buildCaffeine(CaffeineCacheProperties properties) {
+        Caffeine<Object, Object> builder = Caffeine.newBuilder();
+        if (properties.getInitialCapacity() != -1) {
+            builder.initialCapacity(properties.getInitialCapacity());
+        }
+        if (properties.getMaximumSize() != -1) {
+            builder.maximumSize(properties.getMaximumSize());
+        }
+        if (properties.isRecordStats()) {
+            builder.recordStats();
+        }
+
+        if (properties.getExpireAfterWrite() != null) {
+            builder.expireAfterWrite(properties.getExpireAfterWrite());
+        }
+        return builder;
+    }
+
     @Bean
     public IdGenerator<ApplicationUid> applicationUidGenerator() {
         return new RandomApplicationUidGenerator();
     }
 
+
+    //micrometer caffeine cache metrics
+    @Bean
+    public MeterBinder caffeineCacheMeterBinder(@Qualifier("applicationUidCache") CacheManager cacheManager,
+                                                @Qualifier("applicationUidCacheProperties") CaffeineCacheProperties properties) {
+        if (!properties.isRecordStats()) {
+            logger.info("Skipping CaffeineCacheMetrics. recordStats is false");
+            return meterRegistry -> {};
+        }
+        org.springframework.cache.Cache springCache = cacheManager.getCache(APPLICATION_UID_CACHE_NAME);
+        if (springCache == null) {
+            logger.warn("Skipping CaffeineCacheMetrics. cache '{}' not found", APPLICATION_UID_CACHE_NAME);
+            return meterRegistry -> {};
+        }
+
+        Cache<?, ?> nativeCache = (Cache<?, ?>) springCache.getNativeCache();
+        return meterRegistry -> CaffeineCacheMetrics.monitor(meterRegistry, nativeCache, APPLICATION_UID_CACHE_NAME);
+    }
 }

collector/src/main/java/com/navercorp/pinpoint/collector/uid/config/ServiceUidCacheConfig.java

@@ -2,7 +2,7 @@
 
 
 import com.github.benmanes.caffeine.cache.Caffeine;
-import com.navercorp.pinpoint.common.server.uid.cache.CaffeineCacheSpec;
+import com.navercorp.pinpoint.common.server.uid.cache.CaffeineCacheProperties;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.springframework.beans.factory.annotation.Qualifier;
@@ -29,17 +29,32 @@
 
     @Bean
     @ConfigurationProperties(prefix = "collector.service.uid.cache")
-    public CaffeineCacheSpec serviceUidCacheSpec() {
-        return new CaffeineCacheSpec();
+    public CaffeineCacheProperties serviceUidCacheProperties() {
+        return new CaffeineCacheProperties();
     }
 
     @Bean
-    public CacheManager collectorServiceUidCache(@Qualifier("serviceUidCacheSpec") CaffeineCacheSpec caffeineCacheSpec) {
+    public CacheManager collectorServiceUidCache(@Qualifier("serviceUidCacheProperties") CaffeineCacheProperties caffeineCacheProperties) {
         CaffeineCacheManager caffeineCacheManager = new CaffeineCacheManager(SERVICE_UID_CACHE_NAME);
-        caffeineCacheManager.setCaffeine(
-                Caffeine.from(caffeineCacheSpec.getSpecification())
-        );
-
+        caffeineCacheManager.setCaffeine(buildCaffeine(caffeineCacheProperties));
         return caffeineCacheManager;
     }
+
+    private Caffeine<Object, Object> buildCaffeine(CaffeineCacheProperties properties) {
+        Caffeine<Object, Object> builder = Caffeine.newBuilder();
+        if (properties.getInitialCapacity() != -1) {
+            builder.initialCapacity(properties.getInitialCapacity());
+        }
+        if (properties.getMaximumSize() != -1) {
+            builder.maximumSize(properties.getMaximumSize());
+        }
+        if (properties.isRecordStats()) {
+            builder.recordStats();
+        }
+
+        if (properties.getExpireAfterWrite() != null) {
+            builder.expireAfterWrite(properties.getExpireAfterWrite());
+        }
+        return builder;
+    }
 }

collector/src/main/java/com/navercorp/pinpoint/collector/uid/dao/AgentNameDao.java

@@ -0,0 +1,8 @@
+package com.navercorp.pinpoint.collector.uid.dao;
+
+import com.navercorp.pinpoint.common.server.uid.ApplicationUid;
+import com.navercorp.pinpoint.common.server.uid.ServiceUid;
+
+public interface AgentNameDao {
+    void insert(ServiceUid serviceUid, ApplicationUid applicationUid, String agentId, long agentStartTime, String agentName);
+}

collector/src/main/java/com/navercorp/pinpoint/collector/uid/dao/hbase/HbaseAgentNameDao.java

@@ -0,0 +1,44 @@
+package com.navercorp.pinpoint.collector.uid.dao.hbase;
+
+import com.navercorp.pinpoint.collector.uid.dao.AgentNameDao;
+import com.navercorp.pinpoint.common.hbase.HbaseColumnFamily;
+import com.navercorp.pinpoint.common.hbase.HbaseOperations;
+import com.navercorp.pinpoint.common.hbase.HbaseTables;
+import com.navercorp.pinpoint.common.hbase.TableNameProvider;
+import com.navercorp.pinpoint.common.server.uid.ApplicationUid;
+import com.navercorp.pinpoint.common.server.uid.ServiceUid;
+import com.navercorp.pinpoint.common.server.util.AgentListRowKeyUtils;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.stereotype.Repository;
+
+import java.util.Objects;
+
+@Repository
+@ConditionalOnProperty(name = "pinpoint.collector.application.uid.enable", havingValue = "true")
+public class HbaseAgentNameDao implements AgentNameDao {
+
+    private static final HbaseColumnFamily DESCRIPTOR = HbaseTables.AGENT_NAME;
+
+    private final HbaseOperations hbaseOperations;
+    private final TableNameProvider tableNameProvider;
+
+    public HbaseAgentNameDao(HbaseOperations hbaseOperations, TableNameProvider tableNameProvider) {
+        this.hbaseOperations = Objects.requireNonNull(hbaseOperations, "hbaseOperations");
+        this.tableNameProvider = Objects.requireNonNull(tableNameProvider, "tableNameProvider");
+    }
+
+    @Override
+    public void insert(ServiceUid serviceUid, ApplicationUid applicationUid, String agentId, long agentStartTime, String agentName) {
+        byte[] rowKey = AgentListRowKeyUtils.makeRowKey(serviceUid, applicationUid, agentId, agentStartTime);
+        byte[] value = Bytes.toBytes(agentName);
+
+        Put put = new Put(rowKey, true);
+        put.addColumn(DESCRIPTOR.getName(), DESCRIPTOR.getName(), value);
+
+        final TableName agentListTableName = tableNameProvider.getTableName(DESCRIPTOR.getTable());
+        hbaseOperations.put(agentListTableName, put);
+    }
+}

collector/src/main/java/com/navercorp/pinpoint/collector/uid/service/EmptyApplicationUidService.java

@@ -2,8 +2,19 @@
 
 import com.navercorp.pinpoint.common.server.uid.ApplicationUid;
 import com.navercorp.pinpoint.common.server.uid.ServiceUid;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.stereotype.Service;
 
+@Service
+@ConditionalOnProperty(value = "pinpoint.collector.application.uid.enable", havingValue = "false", matchIfMissing = true)
 public class EmptyApplicationUidService implements ApplicationUidService {
+    private final Logger logger = LogManager.getLogger(this.getClass());
+
+    public EmptyApplicationUidService() {
+        logger.info("EmptyApplicationUidService initialized");
+    }
 
     @Override
     public ApplicationUid getApplicationUid(ServiceUid serviceUid, String applicationName) {

collector/src/main/java/com/navercorp/pinpoint/collector/uid/service/StaticServiceGroupService.java

@@ -1,13 +1,22 @@
 package com.navercorp.pinpoint.collector.uid.service;
 
 import com.navercorp.pinpoint.common.server.uid.ServiceUid;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.stereotype.Service;
 
+@Service
+@ConditionalOnProperty(name = "pinpoint.collector.v4.enable", havingValue = "false", matchIfMissing = true)
 public class StaticServiceGroupService implements ServiceGroupService {
+    private final Logger logger = LogManager.getLogger(this.getClass());
 
     private final ServiceUid staticServiceUid;
 
-    public StaticServiceGroupService(int uid) {
+    public StaticServiceGroupService(@Value("${collector.service.uid.default.value:0}") int uid) {
         this.staticServiceUid = ServiceUid.of(uid);
+        logger.info("StaticServiceGroupService initialized. {}", this.staticServiceUid);
     }
 
     @Override

collector/src/main/resources/pinpoint-collector-root.properties

@@ -64,14 +64,16 @@ pinpoint.collector.realtime.atc.enable-count-metric=false
 # Pinpoint Uid (Do not change, under development)
 pinpoint.collector.v4.enable=false
 collector.service.uid.default.value=0
-collector.service.uid.cache.spec.initialCapacity=10
-collector.service.uid.cache.spec.maximumSize=200
-collector.service.uid.cache.spec.expireAfterWrite=300s
+collector.service.uid.cache.initialCapacity=16
+collector.service.uid.cache.maximumSize=200
+collector.service.uid.cache.expireAfterWrite=3600s
 
 pinpoint.collector.application.uid.enable=false
-collector.application.uid.cache.spec.initialCapacity=10
-collector.application.uid.cache.spec.maximumSize=200
-collector.application.uid.cache.spec.expireAfterWrite=300s
+pinpoint.collector.application.uid.agent.list.enable=false
+collector.application.uid.cache.initialCapacity=16
+collector.application.uid.cache.maximumSize=2000
+collector.application.uid.cache.expireAfterWrite=600s
+collector.application.uid.cache.recordStats=false
 
 ###########################################################
 # BANNER                                                  #

commons-hbase/src/main/java/com/navercorp/pinpoint/common/hbase/HbaseTable.java

@@ -32,6 +32,7 @@ public enum HbaseTable {
     APPLICATION_INDEX("ApplicationIndex"),
     APPLICATION_UID("ApplicationUid"),
     APPLICATION_NAME("ApplicationName"),
+    AGENT_NAME("AgentName"),
     APPLICATION_TRACE_INDEX("ApplicationTraceIndex"),
     HOST_APPLICATION_MAP_VER2("HostApplicationMap_Ver2"),
     MAP_STATISTICS_CALLEE_VER2("ApplicationMapStatisticsCallee_Ver2"),

commons-hbase/src/main/java/com/navercorp/pinpoint/common/hbase/HbaseTables.java

@@ -44,6 +44,7 @@ private ApiMetadata(HbaseTable hBaseTable, byte[] columnFamilyName) {
 
     public static final HbaseColumnFamily APPLICATION_NAME = new HbaseColumnFamily(HbaseTable.APPLICATION_NAME, Bytes.toBytes("N"));
 
+    public static final HbaseColumnFamily AGENT_NAME = new HbaseColumnFamily(HbaseTable.AGENT_NAME, Bytes.toBytes("A"));
 
     public static final HbaseColumnFamily APPLICATION_INDEX_AGENTS = new HbaseColumnFamily(HbaseTable.APPLICATION_INDEX, Bytes.toBytes("Agents"));