Prometheus service

README.md

@@ -89,25 +89,28 @@ bin/pgio -D <postgresql data dir>
 ## Options
 
 ```
-Option                   Description                                           
-------                   -----------                                           
--D <String>              Specifies the file system location of the database    
-                           configuration files. If this is omitted, the        
-                           environment variable PGDATA is used.                
--a, --advanced           Enable advanced options                               
--d, --debug              Show debug messages                                   
--g, --group <String>     Group results using specified group configuration file
-                           (advanced option)                                   
--h, --help               Displays this help message and quit                   
--i, --interval <String>  Interval in milliseconds to gather stats (default:    
-                           3000)                                               
---no-print-header        Suppress print of CSV header                          
--o, --show-other         Print read/write data not accounted by any listed     
-                           process                                             
---ppid <String>          Parent pid of the process to scan (advanced option)   
---prometheus-format      Print output in prometheus format                     
--s, --show-system        Print read/write data for the whole system            
--v, --version            Show version and quit                                 
+Option                      Description                                        
+------                      -----------                                        
+-D <String>                 Specifies the file system location of the database 
+                              configuration files. If this is omitted, the     
+                              environment variable PGDATA is used.             
+-a, --advanced              Enable advanced options                            
+-d, --debug                 Show debug messages                                
+-g, --group <String>        Group results using specified group configuration  
+                              file (advanced option)                           
+-h, --help                  Displays this help message and quit                
+-i, --interval <String>     Interval in milliseconds to gather stats (default: 
+                              3000)                                            
+--no-print-header           Suppress print of CSV header                       
+-o, --show-other            Print read/write data not accounted by any listed  
+                              process                                          
+--ppid <String>             Parent pid of the process to scan (advanced option)
+--prometheus-bind <String>  The bind address of prometheus service (advanced   
+                              option)                                          
+--prometheus-port <String>  The port of prometheus service (advanced option)   
+--prometheus-service        Run as a prometheus service (advanced option)      
+-s, --show-system           Print read/write data for the whole system         
+-v, --version               Show version and quit        
 ```
 
 ### Group configuration file
@@ -123,6 +126,10 @@ A JSON indicating groups has the following syntax (regular expression will use p
 }
 ```
 
-## Prometheus
+## Prometheus service
 
-TODO
\ No newline at end of file
+To start pgio as a prometheus service:
+
+```
+bin/pgio -D <postgresql data dir> --advanced --prometheus-service --prometheus-bind 0.0.0.0 --prometheus-port 9090
+```
\ No newline at end of file

main/firefox.json

-{
-  "firefox": [ ".*firefox.*" ]
-}
\ No newline at end of file

main/gitlab-postgresql.json

-{
-  "wal-e": [ "/?(\\w+/)*wal-e .*" ],
-  "wal-e gpg": [ "/?(\\w+/)*wal-e -e -z 0 -r [^ ]+" ],
-  "pgbouncer": [ ".*=pgbouncer gitlabhq_production .*", ".*/var/opt/gitlab/pgbouncer/pgbouncer.ini.*" ],
-  "gitlab-monitor": [ ".*gitlab-monitor.*" ],
-  "archiver": [ ".*archiver.*" ],
-  "wal sender": [ ".*wal sender.*" ],
-  "repmgr": [ ".*gitlab_repmgr gitlab_repmgr.*" ],
-  "query": [ ".*gitlab gitlabhq_production" ],
-  "bgwriter": [ ".*writer process.*" ],
-  "autovacuum": [ ".*autovacuum worker process.*" ],
-  "stats": [ ".*stats collector process.*" ],
-  "wal writer": [ ".*wal writer process.*" ],
-  "checkpoint": [ ".*checkpointer process.*" ]
-}
\ No newline at end of file

main/pom.xml

@@ -32,6 +32,10 @@
         <groupId>org.glassfish</groupId>
         <artifactId>javax.json</artifactId>
     </dependency>
+    <dependency>
+      <groupId>org.nanohttpd</groupId>
+      <artifactId>nanohttpd</artifactId>
+    </dependency>
 
     <dependency>
       <groupId>org.apache.logging.log4j</groupId>

main/src/main/java/com/ongres/pgio/main/Main.java

@@ -27,9 +27,9 @@ import com.ongres.pgio.main.stats.ProcessGroupInfo;
 import com.ongres.pgio.main.stats.StatProcessor;
 import com.ongres.pgio.main.stats.StatSnapshot;
 import com.ongres.pgio.main.stats.serializer.CsvSerializer;
-import com.ongres.pgio.main.stats.serializer.PrometheusSerializer;
 import com.ongres.pgio.main.stats.serializer.StatSerializer;
 import com.ongres.pgio.main.version.Version;
+import fi.iki.elonen.NanoHTTPD;
 import joptsimple.OptionParser;
 import joptsimple.OptionSet;
 
@@ -37,6 +37,8 @@ import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.time.Duration;
@@ -68,26 +70,10 @@ public class Main {
       
       Config config = configOptionSet(options);
       
-      StatSerializer serializer;
-      if (config.isPrometheusFormat()) {
-        serializer = new PrometheusSerializer(System.out);
+      if (config.isPrometheusService()) {
+        runPrometheusService(config);
       } else {
-        serializer = new CsvSerializer(config, System.out);
-      }
-      StatProcessor statProcessor = new StatProcessor(config, serializer, System.err);
-      try {
-        statProcessor.begin();
-        Instant start = Instant.now();
-        Optional<StatSnapshot> previousStats = Optional.empty();
-        while (true) {
-          previousStats = Optional.of(statProcessor.getNext(previousStats));
-          if (config.getInterval() > 0) {
-            Thread.sleep(config.getInterval() - Duration.between(
-                start, Instant.now()).toMillis() % config.getInterval());
-          }
-        }
-      } finally {
-        statProcessor.end();
+        runCollector(config);
       }
     } catch (Throwable throwable) {
       if (throwable.getMessage() != null) {
@@ -104,6 +90,30 @@ public class Main {
     }
   }
 
+  private static void runPrometheusService(Config config) throws IOException {
+    PrometheusService service = new PrometheusService(config);
+    service.start(NanoHTTPD.SOCKET_READ_TIMEOUT, false);
+  }
+
+  private static void runCollector(Config config) throws Exception, InterruptedException {
+    StatSerializer serializer = new CsvSerializer(config, System.out);
+    StatProcessor statProcessor = new StatProcessor(config, serializer, System.err);
+    try {
+      statProcessor.begin();
+      Instant start = Instant.now();
+      Optional<StatSnapshot> previousStats = Optional.empty();
+      while (true) {
+        previousStats = Optional.of(statProcessor.getNext(previousStats));
+        if (config.getInterval() > 0) {
+          Thread.sleep(config.getInterval() - Duration.between(
+              start, Instant.now()).toMillis() % config.getInterval());
+        }
+      }
+    } finally {
+      statProcessor.end();
+    }
+  }
+
   private static OptionParser createOptionParser() {
     OptionParser parser = new OptionParser(false);
     parser.acceptsAll(Lists.newArrayList("h", "help"), 
@@ -129,8 +139,17 @@ public class Main {
         "Suppress print of CSV header");
     parser.acceptsAll(Lists.newArrayList("a", "advanced"), 
         "Enable advanced options");
-    parser.acceptsAll(Lists.newArrayList("prometheus-format"), 
-        "Print output in prometheus format");
+    parser.acceptsAll(Lists.newArrayList("prometheus-service"), 
+        "Run as a prometheus service (advanced option)")
+      .availableIf("advanced");
+    parser.acceptsAll(Lists.newArrayList("prometheus-bind"), 
+        "The bind address of prometheus service (advanced option)")
+      .availableIf("advanced")
+      .withRequiredArg();
+    parser.acceptsAll(Lists.newArrayList("prometheus-port"), 
+        "The port of prometheus service (advanced option)")
+      .availableIf("advanced")
+      .withRequiredArg();
     parser.acceptsAll(Lists.newArrayList("ppid"), 
         "Parent pid of the process to scan (advanced option)")
       .availableIf("advanced")
@@ -155,7 +174,9 @@ public class Main {
     configHelper.setIf("show-system", configBuilder::withShowSystem);
     configHelper.setIf("show-other", configBuilder::withShowOther);
     configHelper.set("group", Main::readGroupConfig, configBuilder::appendProcessGroups);
-    configHelper.setIf("prometheus-format", configBuilder::withPrometheusFormat);
+    configHelper.setIf("prometheus-service", configBuilder::withPrometheusService);
+    configHelper.set("prometheus-bind", Main::readInetAddress, configBuilder::withPrometheusBind);
+    configHelper.set("prometheus-port", Integer::valueOf, configBuilder::withPrometheusPort);
     configHelper.setIf("debug", configBuilder::withDebug);
     
     Config config = configBuilder.build();
@@ -184,6 +205,14 @@ public class Main {
     }
   }
   
+  private static InetAddress readInetAddress(String address) {
+    try {
+      return InetAddress.getByName(address);
+    } catch (UnknownHostException ex) {
+      throw new RuntimeException(ex);
+    }
+  }
+  
   private static int parsePostgresPidFile(Path dataDir) {
     Path postmasterPid = dataDir.resolve("postmaster.pid");
     

main/src/main/java/com/ongres/pgio/main/PrometheusService.java

+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package com.ongres.pgio.main;
+
+import com.ongres.pgio.main.config.Config;
+import com.ongres.pgio.main.stats.StatProcessor;
+import com.ongres.pgio.main.stats.StatSnapshot;
+import com.ongres.pgio.main.stats.serializer.PrometheusSerializer;
+import com.ongres.pgio.main.stats.serializer.StatSerializer;
+import fi.iki.elonen.NanoHTTPD;
+import fi.iki.elonen.NanoHTTPD.Response.Status;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.PrintStream;
+import java.util.Optional;
+import java.util.concurrent.Callable;
+import java.util.concurrent.atomic.AtomicReference;
+
+public class PrometheusService extends NanoHTTPD {
+  private final Config config;
+  private final AtomicReference<Optional<StatSnapshot>> previousStatsReference = 
+      new AtomicReference<Optional<StatSnapshot>>(Optional.empty());
+  
+  public PrometheusService(Config config) {
+    super(config.getPrometheusBind().getHostName(),
+        config.getPrometheusPort());
+    this.config = config;
+  }
+
+  @Override
+  public Response serve(IHTTPSession session) {
+    ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
+    PrintStream out = new PrintStream(byteArrayOutputStream);
+    StatSerializer serializer = new PrometheusSerializer(out);
+    StatProcessor statProcessor = new StatProcessor(config, serializer, System.err);
+    previousStatsReference.getAndUpdate(
+        previousStats -> emptyIfException(() -> statProcessor.getNext(previousStats)));
+    return newFixedLengthResponse(Status.OK, NanoHTTPD.MIME_PLAINTEXT, 
+        new ByteArrayInputStream(byteArrayOutputStream.toByteArray()), 
+        byteArrayOutputStream.size());
+  }
+
+  private static <T> Optional<T> emptyIfException(Callable<T> call) {
+    try {
+      return Optional.of(call.call());
+    } catch (Exception ex) {
+      ex.printStackTrace(System.err);
+      return Optional.empty();
+    }
+  }
+}

main/src/main/java/com/ongres/pgio/main/config/Config.java

@@ -24,6 +24,8 @@ import com.google.common.collect.ImmutableList;
 import com.ongres.pgio.main.stats.ProcessGroupInfo;
 import com.ongres.pgio.main.stats.groups.DefaultPostgresGroups;
 
+import java.net.InetAddress;
+import java.net.UnknownHostException;
 import java.nio.file.Path;
 import java.util.Optional;
 
@@ -32,7 +34,9 @@ public class Config {
   private final Path dataDir;
   private final long interval;
   private final Optional<Integer> ppid;
-  private final boolean prometheusFormat;
+  private final boolean prometheusService;
+  private final InetAddress prometheusBind;
+  private final int prometheusPort;
   private final boolean noPrintHeader;
   private final boolean showSystem;
   private final boolean showOther;
@@ -40,7 +44,7 @@ public class Config {
   private final boolean debug;
 
   private Config(Path dataDir, long interval, Optional<Integer> ppid, 
-      boolean prometheusFormat, 
+      boolean prometheusService, InetAddress prometheusBind, int prometheusPort,
       boolean noPrintHeader, boolean showSystem, boolean showOther,
       Optional<ImmutableList<ProcessGroupInfo>> processGroups,
       boolean debug) {
@@ -48,7 +52,9 @@ public class Config {
     this.dataDir = dataDir;
     this.interval = interval;
     this.ppid = ppid;
-    this.prometheusFormat = prometheusFormat;
+    this.prometheusService = prometheusService;
+    this.prometheusBind = prometheusBind;
+    this.prometheusPort = prometheusPort;
     this.noPrintHeader = noPrintHeader;
     this.showSystem = showSystem;
     this.showOther = showOther;
@@ -68,8 +74,16 @@ public class Config {
     return ppid;
   }
 
-  public boolean isPrometheusFormat() {
-    return prometheusFormat;
+  public boolean isPrometheusService() {
+    return prometheusService;
+  }
+
+  public InetAddress getPrometheusBind() {
+    return prometheusBind;
+  }
+
+  public int getPrometheusPort() {
+    return prometheusPort;
   }
 
   public boolean isNoPrintHeader() {
@@ -96,7 +110,9 @@ public class Config {
     private Path dataDir;
     private long interval;
     private Optional<Integer> ppid = Optional.empty();
-    private boolean prometheusFormat;
+    private boolean prometheusService;
+    private InetAddress prometheusBind;
+    private int prometheusPort = 8000;
     private boolean noPrintHeader;
     private boolean showSystem;
     private boolean showOther;
@@ -104,6 +120,14 @@ public class Config {
         Optional.of(DefaultPostgresGroups.GROUPS);
     private boolean debug;
     
+    public Builder() {
+      try {
+        prometheusBind = InetAddress.getLocalHost();
+      } catch (UnknownHostException ex) {
+        throw new RuntimeException(ex);
+      }
+    }
+    
     public Builder withDataDir(Path dataDir) {
       this.dataDir = dataDir;
       return this;
@@ -119,8 +143,18 @@ public class Config {
       return this;
     }
     
-    public Builder withPrometheusFormat(boolean prometheusFormat) {
-      this.prometheusFormat = prometheusFormat;
+    public Builder withPrometheusService(boolean prometheusService) {
+      this.prometheusService = prometheusService;
+      return this;
+    }
+    
+    public Builder withPrometheusBind(InetAddress prometheusBind) {
+      this.prometheusBind = prometheusBind;
+      return this;
+    }
+    
+    public Builder withPrometheusPort(int prometheusPort) {
+      this.prometheusPort = prometheusPort;
       return this;
     }
     
@@ -164,7 +198,8 @@ public class Config {
           "no database directory specified and environment variable PGDATA unset");
       
       return new Config(dataDir, interval, ppid, 
-          prometheusFormat, noPrintHeader, showSystem, showOther, 
+          prometheusService, prometheusBind, prometheusPort,
+          noPrintHeader, showSystem, showOther, 
           processGroups, debug);
     }
   }

main/src/main/java/com/ongres/pgio/main/stats/serializer/PrometheusSerializer.java

@@ -45,20 +45,27 @@ public class PrometheusSerializer implements StatSerializer {
         ).entrySet()) {
       String label = entry.getKey();
       Optional<UnsignedLong> value = entry.getValue();
+      
+      if (!value.isPresent()) {
+        continue;
+      }
+      
+      out.print("# HELP ");
+      out.print(label);
+      out.print(' ');
+      out.println(label);
       out.print("# TYPE ");
       out.print(label);
       out.println(" gauge");
       out.print(label);
-      out.print(" { label=");
+      out.print("{label=");
       out.print(protectString(stat.getLabel()));
-      out.print(' ');
-      out.print(stringIfPresentOrEmpty(stat.getPid().map(ppid -> "pid=" + ppid + " ")));
-      out.print(stringIfPresentOrEmpty(stat.getPpid().map(ppid -> "ppid=" + ppid + " ")));
+      out.print(stringIfPresentOrEmpty(stat.getPid().map(pid -> " pid=" + pid)));
+      out.print(stringIfPresentOrEmpty(stat.getPpid().map(ppid -> " ppid=" + ppid)));
       out.print("} ");
-      out.print(value);
-      out.print(" [");
-      out.print(stat.getTimestamp().toEpochMilli());
-      out.println(']');
+      out.print(value.get());
+      out.print(' ');
+      out.println(stat.getTimestamp().toEpochMilli());
     }
   }
 

pom.xml

@@ -114,6 +114,11 @@
         <artifactId>javax.json</artifactId>
         <version>1.1.2</version>
       </dependency>
+      <dependency>
+        <groupId>org.nanohttpd</groupId>
+        <artifactId>nanohttpd</artifactId>
+        <version>2.2.0</version>
+      </dependency>
 
 
       <dependency>