Skip to content

GitLab

P
pgio
Commit 5d8be4bf authored by Matteo Melli's avatar Matteo Melli
Browse files
Options

Prometheus service

parent 55ce79bf
Hide whitespace changes
Inline Side-by-side
Showing with 216 additions and 79 deletions
README.md
View file @ 5d8be4bf
......@@ -89,25 +89,28 @@ bin/pgio -D <postgresql data dir>
## Options
```
Option Description
------ -----------
-D <String> Specifies the file system location of the database
configuration files. If this is omitted, the
environment variable PGDATA is used.
-a, --advanced Enable advanced options
-d, --debug Show debug messages
-g, --group <String> Group results using specified group configuration file
(advanced option)
-h, --help Displays this help message and quit
-i, --interval <String> Interval in milliseconds to gather stats (default:
3000)
--no-print-header Suppress print of CSV header
-o, --show-other Print read/write data not accounted by any listed
process
--ppid <String> Parent pid of the process to scan (advanced option)
--prometheus-format Print output in prometheus format
-s, --show-system Print read/write data for the whole system
-v, --version Show version and quit
Option Description
------ -----------
-D <String> Specifies the file system location of the database
configuration files. If this is omitted, the
environment variable PGDATA is used.
-a, --advanced Enable advanced options
-d, --debug Show debug messages
-g, --group <String> Group results using specified group configuration
file (advanced option)
-h, --help Displays this help message and quit
-i, --interval <String> Interval in milliseconds to gather stats (default:
3000)
--no-print-header Suppress print of CSV header
-o, --show-other Print read/write data not accounted by any listed
process
--ppid <String> Parent pid of the process to scan (advanced option)
--prometheus-bind <String> The bind address of prometheus service (advanced
option)
--prometheus-port <String> The port of prometheus service (advanced option)
--prometheus-service Run as a prometheus service (advanced option)
-s, --show-system Print read/write data for the whole system
-v, --version Show version and quit
```
### Group configuration file
......@@ -123,6 +126,10 @@ A JSON indicating groups has the following syntax (regular expression will use p
}
```
## Prometheus
## Prometheus service
TODO
\ No newline at end of file
To start pgio as a prometheus service:
```
bin/pgio -D <postgresql data dir> --advanced --prometheus-service --prometheus-bind 0.0.0.0 --prometheus-port 9090
```
\ No newline at end of file
main/firefox.json deleted 100644 → 0
View file @ 55ce79bf
{
"firefox": [ ".*firefox.*" ]
}
\ No newline at end of file
main/gitlab-postgresql.json deleted 100644 → 0
View file @ 55ce79bf
{
"wal-e": [ "/?(\\w+/)*wal-e .*" ],
"wal-e gpg": [ "/?(\\w+/)*wal-e -e -z 0 -r [^ ]+" ],
"pgbouncer": [ ".*=pgbouncer gitlabhq_production .*", ".*/var/opt/gitlab/pgbouncer/pgbouncer.ini.*" ],
"gitlab-monitor": [ ".*gitlab-monitor.*" ],
"archiver": [ ".*archiver.*" ],
"wal sender": [ ".*wal sender.*" ],
"repmgr": [ ".*gitlab_repmgr gitlab_repmgr.*" ],
"query": [ ".*gitlab gitlabhq_production" ],
"bgwriter": [ ".*writer process.*" ],
"autovacuum": [ ".*autovacuum worker process.*" ],
"stats": [ ".*stats collector process.*" ],
"wal writer": [ ".*wal writer process.*" ],
"checkpoint": [ ".*checkpointer process.*" ]
}
\ No newline at end of file
main/pom.xml
View file @ 5d8be4bf
......@@ -32,6 +32,10 @@
<groupId>org.glassfish</groupId>
<artifactId>javax.json</artifactId>
</dependency>
<dependency>
<groupId>org.nanohttpd</groupId>
<artifactId>nanohttpd</artifactId>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
......
main/src/main/java/com/ongres/pgio/main/Main.java
View file @ 5d8be4bf
......@@ -27,9 +27,9 @@ import com.ongres.pgio.main.stats.ProcessGroupInfo;
import com.ongres.pgio.main.stats.StatProcessor;
import com.ongres.pgio.main.stats.StatSnapshot;
import com.ongres.pgio.main.stats.serializer.CsvSerializer;
import com.ongres.pgio.main.stats.serializer.PrometheusSerializer;
import com.ongres.pgio.main.stats.serializer.StatSerializer;
import com.ongres.pgio.main.version.Version;
import fi.iki.elonen.NanoHTTPD;
import joptsimple.OptionParser;
import joptsimple.OptionSet;
......@@ -37,6 +37,8 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.time.Duration;
......@@ -68,26 +70,10 @@ public class Main {
Config config = configOptionSet(options);
StatSerializer serializer;
if (config.isPrometheusFormat()) {
serializer = new PrometheusSerializer(System.out);
if (config.isPrometheusService()) {
runPrometheusService(config);
} else {
serializer = new CsvSerializer(config, System.out);
}
StatProcessor statProcessor = new StatProcessor(config, serializer, System.err);
try {
statProcessor.begin();
Instant start = Instant.now();
Optional<StatSnapshot> previousStats = Optional.empty();
while (true) {
previousStats = Optional.of(statProcessor.getNext(previousStats));
if (config.getInterval() > 0) {
Thread.sleep(config.getInterval() - Duration.between(
start, Instant.now()).toMillis() % config.getInterval());
}
}
} finally {
statProcessor.end();
runCollector(config);
}
} catch (Throwable throwable) {
if (throwable.getMessage() != null) {
......@@ -104,6 +90,30 @@ public class Main {
}
}
private static void runPrometheusService(Config config) throws IOException {
PrometheusService service = new PrometheusService(config);
service.start(NanoHTTPD.SOCKET_READ_TIMEOUT, false);
}
private static void runCollector(Config config) throws Exception, InterruptedException {
StatSerializer serializer = new CsvSerializer(config, System.out);
StatProcessor statProcessor = new StatProcessor(config, serializer, System.err);
try {
statProcessor.begin();
Instant start = Instant.now();
Optional<StatSnapshot> previousStats = Optional.empty();
while (true) {
previousStats = Optional.of(statProcessor.getNext(previousStats));
if (config.getInterval() > 0) {
Thread.sleep(config.getInterval() - Duration.between(
start, Instant.now()).toMillis() % config.getInterval());
}
}
} finally {
statProcessor.end();
}
}
private static OptionParser createOptionParser() {
OptionParser parser = new OptionParser(false);
parser.acceptsAll(Lists.newArrayList("h", "help"),
......@@ -129,8 +139,17 @@ public class Main {
"Suppress print of CSV header");
parser.acceptsAll(Lists.newArrayList("a", "advanced"),
"Enable advanced options");
parser.acceptsAll(Lists.newArrayList("prometheus-format"),
"Print output in prometheus format");
parser.acceptsAll(Lists.newArrayList("prometheus-service"),
"Run as a prometheus service (advanced option)")
.availableIf("advanced");
parser.acceptsAll(Lists.newArrayList("prometheus-bind"),
"The bind address of prometheus service (advanced option)")
.availableIf("advanced")
.withRequiredArg();
parser.acceptsAll(Lists.newArrayList("prometheus-port"),
"The port of prometheus service (advanced option)")
.availableIf("advanced")
.withRequiredArg();
parser.acceptsAll(Lists.newArrayList("ppid"),
"Parent pid of the process to scan (advanced option)")
.availableIf("advanced")
......@@ -155,7 +174,9 @@ public class Main {
configHelper.setIf("show-system", configBuilder::withShowSystem);
configHelper.setIf("show-other", configBuilder::withShowOther);
configHelper.set("group", Main::readGroupConfig, configBuilder::appendProcessGroups);
configHelper.setIf("prometheus-format", configBuilder::withPrometheusFormat);
configHelper.setIf("prometheus-service", configBuilder::withPrometheusService);
configHelper.set("prometheus-bind", Main::readInetAddress, configBuilder::withPrometheusBind);
configHelper.set("prometheus-port", Integer::valueOf, configBuilder::withPrometheusPort);
configHelper.setIf("debug", configBuilder::withDebug);
Config config = configBuilder.build();
......@@ -184,6 +205,14 @@ public class Main {
}
}
private static InetAddress readInetAddress(String address) {
try {
return InetAddress.getByName(address);
} catch (UnknownHostException ex) {
throw new RuntimeException(ex);
}
}
private static int parsePostgresPidFile(Path dataDir) {
Path postmasterPid = dataDir.resolve("postmaster.pid");
......
main/src/main/java/com/ongres/pgio/main/PrometheusService.java 0 → 100644
View file @ 5d8be4bf
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package com.ongres.pgio.main;
import com.ongres.pgio.main.config.Config;
import com.ongres.pgio.main.stats.StatProcessor;
import com.ongres.pgio.main.stats.StatSnapshot;
import com.ongres.pgio.main.stats.serializer.PrometheusSerializer;
import com.ongres.pgio.main.stats.serializer.StatSerializer;
import fi.iki.elonen.NanoHTTPD;
import fi.iki.elonen.NanoHTTPD.Response.Status;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.PrintStream;
import java.util.Optional;
import java.util.concurrent.Callable;
import java.util.concurrent.atomic.AtomicReference;
public class PrometheusService extends NanoHTTPD {
private final Config config;
private final AtomicReference<Optional<StatSnapshot>> previousStatsReference =
new AtomicReference<Optional<StatSnapshot>>(Optional.empty());
public PrometheusService(Config config) {
super(config.getPrometheusBind().getHostName(),
config.getPrometheusPort());
this.config = config;
}
@Override
public Response serve(IHTTPSession session) {
ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
PrintStream out = new PrintStream(byteArrayOutputStream);
StatSerializer serializer = new PrometheusSerializer(out);
StatProcessor statProcessor = new StatProcessor(config, serializer, System.err);
previousStatsReference.getAndUpdate(
previousStats -> emptyIfException(() -> statProcessor.getNext(previousStats)));
return newFixedLengthResponse(Status.OK, NanoHTTPD.MIME_PLAINTEXT,
new ByteArrayInputStream(byteArrayOutputStream.toByteArray()),
byteArrayOutputStream.size());
}
private static <T> Optional<T> emptyIfException(Callable<T> call) {
try {
return Optional.of(call.call());
} catch (Exception ex) {
ex.printStackTrace(System.err);
return Optional.empty();
}
}
}
main/src/main/java/com/ongres/pgio/main/config/Config.java
View file @ 5d8be4bf
......@@ -24,6 +24,8 @@ import com.google.common.collect.ImmutableList;
import com.ongres.pgio.main.stats.ProcessGroupInfo;
import com.ongres.pgio.main.stats.groups.DefaultPostgresGroups;
import java.net.InetAddress;
import java.net.UnknownHostException;
import java.nio.file.Path;
import java.util.Optional;
......@@ -32,7 +34,9 @@ public class Config {
private final Path dataDir;
private final long interval;
private final Optional<Integer> ppid;
private final boolean prometheusFormat;
private final boolean prometheusService;
private final InetAddress prometheusBind;
private final int prometheusPort;
private final boolean noPrintHeader;
private final boolean showSystem;
private final boolean showOther;
......@@ -40,7 +44,7 @@ public class Config {
private final boolean debug;
private Config(Path dataDir, long interval, Optional<Integer> ppid,
boolean prometheusFormat,
boolean prometheusService, InetAddress prometheusBind, int prometheusPort,
boolean noPrintHeader, boolean showSystem, boolean showOther,
Optional<ImmutableList<ProcessGroupInfo>> processGroups,
boolean debug) {
......@@ -48,7 +52,9 @@ public class Config {
this.dataDir = dataDir;
this.interval = interval;
this.ppid = ppid;
this.prometheusFormat = prometheusFormat;
this.prometheusService = prometheusService;
this.prometheusBind = prometheusBind;
this.prometheusPort = prometheusPort;
this.noPrintHeader = noPrintHeader;
this.showSystem = showSystem;
this.showOther = showOther;
......@@ -68,8 +74,16 @@ public class Config {
return ppid;
}
public boolean isPrometheusFormat() {
return prometheusFormat;
public boolean isPrometheusService() {
return prometheusService;
}
public InetAddress getPrometheusBind() {
return prometheusBind;
}
public int getPrometheusPort() {
return prometheusPort;
}
public boolean isNoPrintHeader() {
......@@ -96,7 +110,9 @@ public class Config {
private Path dataDir;
private long interval;
private Optional<Integer> ppid = Optional.empty();
private boolean prometheusFormat;
private boolean prometheusService;
private InetAddress prometheusBind;
private int prometheusPort = 8000;
private boolean noPrintHeader;
private boolean showSystem;
private boolean showOther;
......@@ -104,6 +120,14 @@ public class Config {
Optional.of(DefaultPostgresGroups.GROUPS);
private boolean debug;
public Builder() {
try {
prometheusBind = InetAddress.getLocalHost();
} catch (UnknownHostException ex) {
throw new RuntimeException(ex);
}
}
public Builder withDataDir(Path dataDir) {
this.dataDir = dataDir;
return this;
......@@ -119,8 +143,18 @@ public class Config {
return this;
}
public Builder withPrometheusFormat(boolean prometheusFormat) {
this.prometheusFormat = prometheusFormat;
public Builder withPrometheusService(boolean prometheusService) {
this.prometheusService = prometheusService;
return this;
}
public Builder withPrometheusBind(InetAddress prometheusBind) {
this.prometheusBind = prometheusBind;
return this;
}
public Builder withPrometheusPort(int prometheusPort) {
this.prometheusPort = prometheusPort;
return this;
}
......@@ -164,7 +198,8 @@ public class Config {
"no database directory specified and environment variable PGDATA unset");
return new Config(dataDir, interval, ppid,
prometheusFormat, noPrintHeader, showSystem, showOther,
prometheusService, prometheusBind, prometheusPort,
noPrintHeader, showSystem, showOther,
processGroups, debug);
}
}
......
main/src/main/java/com/ongres/pgio/main/stats/serializer/PrometheusSerializer.java
View file @ 5d8be4bf
......@@ -45,20 +45,27 @@ public class PrometheusSerializer implements StatSerializer {
).entrySet()) {
String label = entry.getKey();
Optional<UnsignedLong> value = entry.getValue();
if (!value.isPresent()) {
continue;
}
out.print("# HELP ");
out.print(label);
out.print(' ');
out.println(label);
out.print("# TYPE ");
out.print(label);
out.println(" gauge");
out.print(label);
out.print(" { label=");
out.print("{label=");
out.print(protectString(stat.getLabel()));
out.print(' ');
out.print(stringIfPresentOrEmpty(stat.getPid().map(ppid -> "pid=" + ppid + " ")));
out.print(stringIfPresentOrEmpty(stat.getPpid().map(ppid -> "ppid=" + ppid + " ")));
out.print(stringIfPresentOrEmpty(stat.getPid().map(pid -> " pid=" + pid)));
out.print(stringIfPresentOrEmpty(stat.getPpid().map(ppid -> " ppid=" + ppid)));
out.print("} ");
out.print(value);
out.print(" [");
out.print(stat.getTimestamp().toEpochMilli());
out.println(']');
out.print(value.get());
out.print(' ');
out.println(stat.getTimestamp().toEpochMilli());
}
}
......
pom.xml
View file @ 5d8be4bf
......@@ -114,6 +114,11 @@
<artifactId>javax.json</artifactId>
<version>1.1.2</version>
</dependency>
<dependency>
<groupId>org.nanohttpd</groupId>
<artifactId>nanohttpd</artifactId>
<version>2.2.0</version>
</dependency>
<dependency>
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment