PGPainless/pgpainless
PGPainless/pgpainless
1
0
Fork 0
mirror of https://github.com/pgpainless/pgpainless.git synced 2025-09-14 20:59:39 +02:00
Code Issues Releases Wiki Activity

SubkeyLookup: Switch to batch inserting

Browse source
This commit is contained in:
Paul Schaub 2022-02-25 17:47:42 +01:00
parent d53d62a6e9
commit 1495baaf6e
Signed by: vanitasvitae
GPG key ID: 62BEE9264BF17311
9 changed files with 110 additions and 86 deletions
Download patch file Download diff file Expand all files Collapse all files

27
pgp-cert-d-java-jdbc-sqlite-lookup/src/main/java/pgp/cert_d/jdbc/sqlite/Entry.java
View file

@ -4,27 +4,44 @@
package pgp.cert_d.jdbc.sqlite; package pgp.cert_d.jdbc.sqlite;
/**
* Subkey-ID database entry.
*/
public class Entry { public class Entry {
private final int id; private final int id;
private final String identifier; private final String certificate;
private final long subkeyId; private final long subkeyId;
public Entry(int id, long subkeyId, String identifier) { public Entry(int id, long subkeyId, String certificate) {
this.id = id; this.id = id;
this.subkeyId = subkeyId; this.subkeyId = subkeyId;
this.identifier = identifier; this.certificate = certificate;
} }
/**
* Get the internal ID of this entry in the database.
*
* @return internal id
*/
public int getId() { public int getId() {
return id; return id;
} }
/**
* Return the key-ID of the subkey.
*
* @return subkey id
*/
public long getSubkeyId() { public long getSubkeyId() {
return subkeyId; return subkeyId;
} }
public String getIdentifier() { /**
return identifier; * Return the fingerprint of the certificate the subkey belongs to.
* @return fingerprint
*/
public String getCertificate() {
return certificate;
} }
} }

59
pgp-cert-d-java-jdbc-sqlite-lookup/src/main/java/pgp/cert_d/jdbc/sqlite/SqliteSubkeyLookup.java
View file

@ -28,16 +28,18 @@ public class SqliteSubkeyLookup implements SubkeyLookup {
private static final String CREATE_TABLE_STMT = "" + private static final String CREATE_TABLE_STMT = "" +
"CREATE TABLE IF NOT EXISTS subkey_lookup (\n" + "CREATE TABLE IF NOT EXISTS subkey_lookup (\n" +
" id integer PRIMARY KEY,\n" + " id integer PRIMARY KEY,\n" + // id (internal to the database)
" identifier text NOT NULL,\n" + " certificate text NOT NULL,\n" + // certificate fingerprint
" subkey_id integer NOT NULL,\n" + " subkey_id integer NOT NULL,\n" + // subkey id
" UNIQUE(identifier, subkey_id)\n" + " UNIQUE(certificate, subkey_id)\n" +
")"; ")";
private static final String INSERT_STMT = "" + private static final String INSERT_STMT = "" +
"INSERT INTO subkey_lookup(identifier, subkey_id) VALUES (?,?)"; "INSERT INTO subkey_lookup(certificate, subkey_id) " +
"VALUES (?,?)";
private static final String QUERY_STMT = "" + private static final String QUERY_STMT = "" +
"SELECT * FROM subkey_lookup WHERE subkey_id=?"; "SELECT * FROM subkey_lookup " +
"WHERE subkey_id=?";
public SqliteSubkeyLookup(String databaseURL) throws SQLException { public SqliteSubkeyLookup(String databaseURL) throws SQLException {
this.databaseUrl = databaseURL; this.databaseUrl = databaseURL;
@ -54,21 +56,27 @@ public class SqliteSubkeyLookup implements SubkeyLookup {
return new SqliteSubkeyLookup("jdbc:sqlite:" + databaseFile.getAbsolutePath()); return new SqliteSubkeyLookup("jdbc:sqlite:" + databaseFile.getAbsolutePath());
} }
public void insert(long subkeyId, String identifier, boolean ignoreDuplicates) throws SQLException { public void insertValues(String certificate, List<Long> subkeyIds) throws SQLException {
try (Connection connection = getConnection(); PreparedStatement statement = connection.prepareStatement(INSERT_STMT)) { try (Connection connection = getConnection(); PreparedStatement statement = connection.prepareStatement(INSERT_STMT)) {
statement.setString(1, identifier); for (long subkeyId : subkeyIds) {
statement.setLong(2, subkeyId); try {
statement.executeUpdate(); statement.setString(1, certificate);
} catch (SQLiteException e) { statement.setLong(2, subkeyId);
// throw any exception, except: statement.executeUpdate();
// ignore unique constraint-related exceptions if we ignoreDuplicates } catch (SQLiteException e) {
if (!ignoreDuplicates || e.getResultCode().code != SQLiteErrorCode.SQLITE_CONSTRAINT_UNIQUE.code) { // throw any exception, except:
throw e; // ignore unique constraint-related exceptions if we ignoreDuplicates
if (e.getResultCode().code == SQLiteErrorCode.SQLITE_CONSTRAINT_UNIQUE.code) {
// ignore duplicates
} else {
throw e;
}
}
} }
} }
} }
public List<Entry> get(long subkeyId) throws SQLException { public List<Entry> selectValues(long subkeyId) throws SQLException {
List<Entry> results = new ArrayList<>(); List<Entry> results = new ArrayList<>();
try (Connection connection = getConnection(); PreparedStatement statement = connection.prepareStatement(QUERY_STMT)) { try (Connection connection = getConnection(); PreparedStatement statement = connection.prepareStatement(QUERY_STMT)) {
statement.setLong(1, subkeyId); statement.setLong(1, subkeyId);
@ -77,7 +85,7 @@ public class SqliteSubkeyLookup implements SubkeyLookup {
Entry entry = new Entry( Entry entry = new Entry(
resultSet.getInt("id"), resultSet.getInt("id"),
resultSet.getLong("subkey_id"), resultSet.getLong("subkey_id"),
resultSet.getString("identifier")); resultSet.getString("certificate"));
results.add(entry); results.add(entry);
} }
} }
@ -86,27 +94,26 @@ public class SqliteSubkeyLookup implements SubkeyLookup {
} }
@Override @Override
public Set<String> getIdentifiersForSubkeyId(long subkeyId) throws IOException { public Set<String> getCertificatesForSubkeyId(long subkeyId) throws IOException {
try { try {
List<Entry> entries = get(subkeyId); List<Entry> entries = selectValues(subkeyId);
Set<String> identifiers = new HashSet<>(); Set<String> certificates = new HashSet<>();
for (Entry entry : entries) { for (Entry entry : entries) {
identifiers.add(entry.getIdentifier()); certificates.add(entry.getCertificate());
} }
return Collections.unmodifiableSet(identifiers); return Collections.unmodifiableSet(certificates);
} catch (SQLException e) { } catch (SQLException e) {
throw new IOException("Cannot query for subkey lookup entries.", e); throw new IOException("Cannot query for subkey lookup entries.", e);
} }
} }
@Override @Override
public void storeIdentifierForSubkeyId(long subkeyId, String identifier) public void storeCertificateSubkeyIds(String certificate, List<Long> subkeyIds) throws IOException {
throws IOException {
try { try {
insert(subkeyId, identifier, true); insertValues(certificate, subkeyIds);
} catch (SQLException e) { } catch (SQLException e) {
throw new IOException("Cannot store subkey lookup entry in database.", e); throw new IOException("Cannot store subkey lookup entries in database.", e);
} }
} }
} }

2
pgp-cert-d-java-jdbc-sqlite-lookup/src/test/java/pgp/cert_d/jdbc/sqlite/EntryTest.java
View file

@ -16,6 +16,6 @@ public class EntryTest {
assertEquals(1, entry.getId()); assertEquals(1, entry.getId());
assertEquals(123L, entry.getSubkeyId()); assertEquals(123L, entry.getSubkeyId());
assertEquals("eb85bb5fa33a75e15e944e63f231550c4f47e38e", entry.getIdentifier()); assertEquals("eb85bb5fa33a75e15e944e63f231550c4f47e38e", entry.getCertificate());
} }
} }

28
pgp-cert-d-java-jdbc-sqlite-lookup/src/test/java/pgp/cert_d/jdbc/sqlite/SqliteSubkeyLookupTest.java
View file

@ -5,7 +5,6 @@
package pgp.cert_d.jdbc.sqlite; package pgp.cert_d.jdbc.sqlite;
import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.File; import java.io.File;
@ -34,40 +33,35 @@ public class SqliteSubkeyLookupTest {
@Test @Test
public void simpleInsertAndGet() throws IOException { public void simpleInsertAndGet() throws IOException {
lookup.storeIdentifierForSubkeyId(123L, "eb85bb5fa33a75e15e944e63f231550c4f47e38e"); lookup.storeCertificateSubkeyIds("eb85bb5fa33a75e15e944e63f231550c4f47e38e", Arrays.asList(123L, 234L));
lookup.storeIdentifierForSubkeyId(234L, "eb85bb5fa33a75e15e944e63f231550c4f47e38e"); lookup.storeCertificateSubkeyIds("d1a66e1a23b182c9980f788cfbfcc82a015e7330", Collections.singletonList(234L));
lookup.storeIdentifierForSubkeyId(234L, "d1a66e1a23b182c9980f788cfbfcc82a015e7330");
assertEquals(Collections.singleton("eb85bb5fa33a75e15e944e63f231550c4f47e38e"), lookup.getIdentifiersForSubkeyId(123L)); assertEquals(Collections.singleton("eb85bb5fa33a75e15e944e63f231550c4f47e38e"), lookup.getCertificatesForSubkeyId(123L));
assertEquals( assertEquals(
new HashSet<>(Arrays.asList("eb85bb5fa33a75e15e944e63f231550c4f47e38e", "d1a66e1a23b182c9980f788cfbfcc82a015e7330")), new HashSet<>(Arrays.asList("eb85bb5fa33a75e15e944e63f231550c4f47e38e", "d1a66e1a23b182c9980f788cfbfcc82a015e7330")),
lookup.getIdentifiersForSubkeyId(234L)); lookup.getCertificatesForSubkeyId(234L));
} }
@Test @Test
public void getNonExistingSubkeyYieldsNull() throws IOException, SQLException { public void getNonExistingSubkeyYieldsNull() throws IOException, SQLException {
assertTrue(lookup.get(6666666).isEmpty()); assertTrue(lookup.selectValues(6666666).isEmpty());
assertTrue(lookup.getIdentifiersForSubkeyId(6666666).isEmpty()); assertTrue(lookup.getCertificatesForSubkeyId(6666666).isEmpty());
} }
@Test @Test
public void secondInstanceLookupTest() throws IOException, SQLException { public void secondInstanceLookupTest() throws IOException, SQLException {
lookup.storeIdentifierForSubkeyId(1337, "eb85bb5fa33a75e15e944e63f231550c4f47e38e"); lookup.storeCertificateSubkeyIds("eb85bb5fa33a75e15e944e63f231550c4f47e38e", Collections.singletonList(1337L));
assertEquals(Collections.singleton("eb85bb5fa33a75e15e944e63f231550c4f47e38e"), lookup.getIdentifiersForSubkeyId(1337)); assertEquals(Collections.singleton("eb85bb5fa33a75e15e944e63f231550c4f47e38e"), lookup.getCertificatesForSubkeyId(1337));
// do the lookup using a second db instance on the same file // do the lookup using a second db instance on the same file
SqliteSubkeyLookup secondInstance = SqliteSubkeyLookup.forDatabaseFile(databaseFile); SqliteSubkeyLookup secondInstance = SqliteSubkeyLookup.forDatabaseFile(databaseFile);
assertEquals(Collections.singleton("eb85bb5fa33a75e15e944e63f231550c4f47e38e"), secondInstance.getIdentifiersForSubkeyId(1337)); assertEquals(Collections.singleton("eb85bb5fa33a75e15e944e63f231550c4f47e38e"), secondInstance.getCertificatesForSubkeyId(1337));
} }
@Test @Test
public void ignoreInsertDuplicates() throws IOException { public void ignoreInsertDuplicates() throws IOException {
lookup.storeIdentifierForSubkeyId(123L, "d1a66e1a23b182c9980f788cfbfcc82a015e7330"); lookup.storeCertificateSubkeyIds("d1a66e1a23b182c9980f788cfbfcc82a015e7330", Arrays.asList(123L, 234L));
// per default we ignore duplicates // per default we ignore duplicates
lookup.storeIdentifierForSubkeyId(123L, "d1a66e1a23b182c9980f788cfbfcc82a015e7330"); lookup.storeCertificateSubkeyIds("d1a66e1a23b182c9980f788cfbfcc82a015e7330", Arrays.asList(123L, 512L));
// if we choose not to ignore duplicates, we raise an exception
assertThrows(SQLException.class, () ->
lookup.insert(123L, "d1a66e1a23b182c9980f788cfbfcc82a015e7330", false));
} }
} }

21
pgp-cert-d-java/src/main/java/pgp/cert_d/InMemorySubkeyLookup.java
View file

@ -4,20 +4,21 @@
package pgp.cert_d; package pgp.cert_d;
import pgp.certificate_store.SubkeyLookup;
import java.util.Collections; import java.util.Collections;
import java.util.HashMap; import java.util.HashMap;
import java.util.HashSet; import java.util.HashSet;
import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Set; import java.util.Set;
import pgp.certificate_store.SubkeyLookup;
public class InMemorySubkeyLookup implements SubkeyLookup { public class InMemorySubkeyLookup implements SubkeyLookup {
private static final Map<Long, Set<String>> subkeyMap = new HashMap<>(); private static final Map<Long, Set<String>> subkeyMap = new HashMap<>();
@Override @Override
public Set<String> getIdentifiersForSubkeyId(long subkeyId) { public Set<String> getCertificatesForSubkeyId(long subkeyId) {
Set<String> identifiers = subkeyMap.get(subkeyId); Set<String> identifiers = subkeyMap.get(subkeyId);
if (identifiers == null) { if (identifiers == null) {
return Collections.emptySet(); return Collections.emptySet();
@ -26,13 +27,15 @@ public class InMemorySubkeyLookup implements SubkeyLookup {
} }
@Override @Override
public void storeIdentifierForSubkeyId(long subkeyId, String identifier) { public void storeCertificateSubkeyIds(String certificate, List<Long> subkeyIds) {
Set<String> identifiers = subkeyMap.get(subkeyId); for (long subkeyId : subkeyIds) {
if (identifiers == null) { Set<String> certificates = subkeyMap.get(subkeyId);
identifiers = new HashSet<>(); if (certificates == null) {
subkeyMap.put(subkeyId, identifiers); certificates = new HashSet<>();
subkeyMap.put(subkeyId, certificates);
}
certificates.add(certificate);
} }
identifiers.add(identifier);
} }
public void clear() { public void clear() {

28
pgp-cert-d-java/src/test/java/pgp/cert_d/SubkeyLookupTest.java
View file

@ -55,33 +55,33 @@ public class SubkeyLookupTest {
public void testInsertGet(SubkeyLookup subject) throws IOException { public void testInsertGet(SubkeyLookup subject) throws IOException {
// Initially all null // Initially all null
assertTrue(subject.getIdentifiersForSubkeyId(123).isEmpty()); assertTrue(subject.getCertificatesForSubkeyId(123).isEmpty());
assertTrue(subject.getIdentifiersForSubkeyId(1337).isEmpty()); assertTrue(subject.getCertificatesForSubkeyId(1337).isEmpty());
assertTrue(subject.getIdentifiersForSubkeyId(420).isEmpty()); assertTrue(subject.getCertificatesForSubkeyId(420).isEmpty());
// Store one val, others still null // Store one val, others still null
subject.storeIdentifierForSubkeyId(123, "d1a66e1a23b182c9980f788cfbfcc82a015e7330"); subject.storeCertificateSubkeyIds("d1a66e1a23b182c9980f788cfbfcc82a015e7330", Collections.singletonList(123L));
assertEquals(Collections.singleton("d1a66e1a23b182c9980f788cfbfcc82a015e7330"), subject.getIdentifiersForSubkeyId(123)); assertEquals(Collections.singleton("d1a66e1a23b182c9980f788cfbfcc82a015e7330"), subject.getCertificatesForSubkeyId(123));
assertTrue(subject.getIdentifiersForSubkeyId(1337).isEmpty()); assertTrue(subject.getCertificatesForSubkeyId(1337).isEmpty());
assertTrue(subject.getIdentifiersForSubkeyId(420).isEmpty()); assertTrue(subject.getCertificatesForSubkeyId(420).isEmpty());
// Store other val, first stays intact // Store other val, first stays intact
subject.storeIdentifierForSubkeyId(1337, "d1a66e1a23b182c9980f788cfbfcc82a015e7330"); subject.storeCertificateSubkeyIds("d1a66e1a23b182c9980f788cfbfcc82a015e7330", Collections.singletonList(1337L));
subject.storeIdentifierForSubkeyId(420, "d1a66e1a23b182c9980f788cfbfcc82a015e7330"); subject.storeCertificateSubkeyIds("d1a66e1a23b182c9980f788cfbfcc82a015e7330", Collections.singletonList(420L));
assertEquals(Collections.singleton("d1a66e1a23b182c9980f788cfbfcc82a015e7330"), subject.getIdentifiersForSubkeyId(123)); assertEquals(Collections.singleton("d1a66e1a23b182c9980f788cfbfcc82a015e7330"), subject.getCertificatesForSubkeyId(123));
assertEquals(Collections.singleton("d1a66e1a23b182c9980f788cfbfcc82a015e7330"), subject.getIdentifiersForSubkeyId(1337)); assertEquals(Collections.singleton("d1a66e1a23b182c9980f788cfbfcc82a015e7330"), subject.getCertificatesForSubkeyId(1337));
assertEquals(Collections.singleton("d1a66e1a23b182c9980f788cfbfcc82a015e7330"), subject.getIdentifiersForSubkeyId(420)); assertEquals(Collections.singleton("d1a66e1a23b182c9980f788cfbfcc82a015e7330"), subject.getCertificatesForSubkeyId(420));
// add additional entry for subkey // add additional entry for subkey
subject.storeIdentifierForSubkeyId(123, "eb85bb5fa33a75e15e944e63f231550c4f47e38e"); subject.storeCertificateSubkeyIds("eb85bb5fa33a75e15e944e63f231550c4f47e38e", Collections.singletonList(123L));
assertEquals( assertEquals(
new HashSet<>(Arrays.asList("eb85bb5fa33a75e15e944e63f231550c4f47e38e", "d1a66e1a23b182c9980f788cfbfcc82a015e7330")), new HashSet<>(Arrays.asList("eb85bb5fa33a75e15e944e63f231550c4f47e38e", "d1a66e1a23b182c9980f788cfbfcc82a015e7330")),
subject.getIdentifiersForSubkeyId(123)); subject.getCertificatesForSubkeyId(123));
} }
} }

13
pgp-certificate-store/src/main/java/pgp/certificate_store/SubkeyLookup.java
View file

@ -5,6 +5,7 @@
package pgp.certificate_store; package pgp.certificate_store;
import java.io.IOException; import java.io.IOException;
import java.util.List;
import java.util.Set; import java.util.Set;
public interface SubkeyLookup { public interface SubkeyLookup {
@ -16,13 +17,15 @@ public interface SubkeyLookup {
* @param subkeyId subkey id * @param subkeyId subkey id
* @return fingerprint of the certificate * @return fingerprint of the certificate
*/ */
Set<String> getIdentifiersForSubkeyId(long subkeyId) throws IOException; Set<String> getCertificatesForSubkeyId(long subkeyId) throws IOException;
/** /**
* Store a record of the subkey id that points to the fingerprint. * Record, which certificate the subkey-ids in the list belong to.
* This method does not change the affiliation of subkey-ids not contained in the provided list.
* *
* @param subkeyId subkey id * @param certificate certificate fingerprint
* @param identifier fingerprint of the certificate * @param subkeyIds subkey ids
* @throws IOException in case of an IO error
*/ */
void storeIdentifierForSubkeyId(long subkeyId, String identifier) throws IOException; void storeCertificateSubkeyIds(String certificate, List<Long> subkeyIds) throws IOException;
} }

14
pgpainless-cert-d/src/main/java/org/pgpainless/certificate_store/SharedPGPCertificateDirectoryAdapter.java
View file

@ -6,7 +6,9 @@ package org.pgpainless.certificate_store;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.util.ArrayList;
import java.util.Iterator; import java.util.Iterator;
import java.util.List;
import java.util.Set; import java.util.Set;
import pgp.cert_d.SharedPGPCertificateDirectory; import pgp.cert_d.SharedPGPCertificateDirectory;
@ -106,18 +108,16 @@ public class SharedPGPCertificateDirectoryAdapter
return; return;
} }
String fingerprint = certificate.getFingerprint(); String fingerprint = certificate.getFingerprint();
for (Long subkeyId : certificate.getSubkeyIds()) { storeCertificateSubkeyIds(fingerprint, new ArrayList<>(certificate.getSubkeyIds()));
storeIdentifierForSubkeyId(subkeyId, fingerprint);
}
} }
@Override @Override
public Set<String> getIdentifiersForSubkeyId(long subkeyId) throws IOException { public Set<String> getCertificatesForSubkeyId(long subkeyId) throws IOException {
return subkeyLookup.getIdentifiersForSubkeyId(subkeyId); return subkeyLookup.getCertificatesForSubkeyId(subkeyId);
} }
@Override @Override
public void storeIdentifierForSubkeyId(long subkeyId, String identifier) throws IOException { public void storeCertificateSubkeyIds(String certificate, List<Long> subkeyIds) throws IOException {
subkeyLookup.storeIdentifierForSubkeyId(subkeyId, identifier); subkeyLookup.storeCertificateSubkeyIds(certificate, subkeyIds);
} }
} }

4
pgpainless-cert-d/src/test/java/org/pgpainless/cert_d/SharedPGPCertificateDirectoryAdapterTest.java
View file

@ -84,7 +84,7 @@ public class SharedPGPCertificateDirectoryAdapterTest {
Set<Long> subkeys = certificate.getSubkeyIds(); Set<Long> subkeys = certificate.getSubkeyIds();
assertEquals(expectedSubkeys, subkeys); assertEquals(expectedSubkeys, subkeys);
for (long subkey : subkeys) { for (long subkey : subkeys) {
assertEquals(Collections.singleton(fingerprint), store.getIdentifiersForSubkeyId(subkey)); assertEquals(Collections.singleton(fingerprint), store.getCertificatesForSubkeyId(subkey));
} }
Certificate retrieved = store.getCertificate(fingerprint); Certificate retrieved = store.getCertificate(fingerprint);
@ -108,7 +108,7 @@ public class SharedPGPCertificateDirectoryAdapterTest {
Set<Long> subkeys = certificate.getSubkeyIds(); Set<Long> subkeys = certificate.getSubkeyIds();
assertEquals(3, subkeys.size()); assertEquals(3, subkeys.size());
for (long subkey : subkeys) { for (long subkey : subkeys) {
assertEquals(Collections.singleton(fingerprint), store.getIdentifiersForSubkeyId(subkey)); assertEquals(Collections.singleton(fingerprint), store.getCertificatesForSubkeyId(subkey));
} }
Certificate retrieved = store.getCertificate(fingerprint); Certificate retrieved = store.getCertificate(fingerprint);