PGPainless/pgpainless
PGPainless/pgpainless
1
0
Fork 0
mirror of https://github.com/pgpainless/pgpainless.git synced 2025-09-14 20:59:39 +02:00
Code Issues Releases Wiki Activity

SubkeyLookup: Switch to batch inserting

Browse source
This commit is contained in:
Paul Schaub 2022-02-25 17:47:42 +01:00
parent d53d62a6e9
commit 1495baaf6e
Signed by: vanitasvitae
GPG key ID: 62BEE9264BF17311
9 changed files with 110 additions and 86 deletions
Download patch file Download diff file Expand all files Collapse all files

27
pgp-cert-d-java-jdbc-sqlite-lookup/src/main/java/pgp/cert_d/jdbc/sqlite/Entry.java
View file

@ -4,27 +4,44 @@
package pgp.cert_d.jdbc.sqlite;
/**
* Subkey-ID database entry.
*/
public class Entry {
private final int id;
private final String identifier;
private final String certificate;
private final long subkeyId;
public Entry(int id, long subkeyId, String identifier) {
public Entry(int id, long subkeyId, String certificate) {
this.id = id;
this.subkeyId = subkeyId;
this.identifier = identifier;
this.certificate = certificate;
}
/**
* Get the internal ID of this entry in the database.
*
* @return internal id
*/
public int getId() {
return id;
}
/**
* Return the key-ID of the subkey.
*
* @return subkey id
*/
public long getSubkeyId() {
return subkeyId;
}
public String getIdentifier() {
return identifier;
/**
* Return the fingerprint of the certificate the subkey belongs to.
* @return fingerprint
*/
public String getCertificate() {
return certificate;
}
}

59
pgp-cert-d-java-jdbc-sqlite-lookup/src/main/java/pgp/cert_d/jdbc/sqlite/SqliteSubkeyLookup.java
View file

@ -28,16 +28,18 @@ public class SqliteSubkeyLookup implements SubkeyLookup {
private static final String CREATE_TABLE_STMT = "" +
"CREATE TABLE IF NOT EXISTS subkey_lookup (\n" +
" id integer PRIMARY KEY,\n" +
" identifier text NOT NULL,\n" +
" subkey_id integer NOT NULL,\n" +
" UNIQUE(identifier, subkey_id)\n" +
" id integer PRIMARY KEY,\n" + // id (internal to the database)
" certificate text NOT NULL,\n" + // certificate fingerprint
" subkey_id integer NOT NULL,\n" + // subkey id
" UNIQUE(certificate, subkey_id)\n" +
")";
private static final String INSERT_STMT = "" +
"INSERT INTO subkey_lookup(identifier, subkey_id) VALUES (?,?)";
"INSERT INTO subkey_lookup(certificate, subkey_id) " +
"VALUES (?,?)";
private static final String QUERY_STMT = "" +
"SELECT * FROM subkey_lookup WHERE subkey_id=?";
"SELECT * FROM subkey_lookup " +
"WHERE subkey_id=?";
public SqliteSubkeyLookup(String databaseURL) throws SQLException {
this.databaseUrl = databaseURL;
@ -54,21 +56,27 @@ public class SqliteSubkeyLookup implements SubkeyLookup {
return new SqliteSubkeyLookup("jdbc:sqlite:" + databaseFile.getAbsolutePath());
}
public void insert(long subkeyId, String identifier, boolean ignoreDuplicates) throws SQLException {
public void insertValues(String certificate, List<Long> subkeyIds) throws SQLException {
try (Connection connection = getConnection(); PreparedStatement statement = connection.prepareStatement(INSERT_STMT)) {
statement.setString(1, identifier);
statement.setLong(2, subkeyId);
statement.executeUpdate();
} catch (SQLiteException e) {
// throw any exception, except:
// ignore unique constraint-related exceptions if we ignoreDuplicates
if (!ignoreDuplicates || e.getResultCode().code != SQLiteErrorCode.SQLITE_CONSTRAINT_UNIQUE.code) {
throw e;
for (long subkeyId : subkeyIds) {
try {
statement.setString(1, certificate);
statement.setLong(2, subkeyId);
statement.executeUpdate();
} catch (SQLiteException e) {
// throw any exception, except:
// ignore unique constraint-related exceptions if we ignoreDuplicates
if (e.getResultCode().code == SQLiteErrorCode.SQLITE_CONSTRAINT_UNIQUE.code) {
// ignore duplicates
} else {
throw e;
}
}
}
}
}
public List<Entry> get(long subkeyId) throws SQLException {
public List<Entry> selectValues(long subkeyId) throws SQLException {
List<Entry> results = new ArrayList<>();
try (Connection connection = getConnection(); PreparedStatement statement = connection.prepareStatement(QUERY_STMT)) {
statement.setLong(1, subkeyId);
@ -77,7 +85,7 @@ public class SqliteSubkeyLookup implements SubkeyLookup {
Entry entry = new Entry(
resultSet.getInt("id"),
resultSet.getLong("subkey_id"),
resultSet.getString("identifier"));
resultSet.getString("certificate"));
results.add(entry);
}
}
@ -86,27 +94,26 @@ public class SqliteSubkeyLookup implements SubkeyLookup {
}
@Override
public Set<String> getIdentifiersForSubkeyId(long subkeyId) throws IOException {
public Set<String> getCertificatesForSubkeyId(long subkeyId) throws IOException {
try {
List<Entry> entries = get(subkeyId);
Set<String> identifiers = new HashSet<>();
List<Entry> entries = selectValues(subkeyId);
Set<String> certificates = new HashSet<>();
for (Entry entry : entries) {
identifiers.add(entry.getIdentifier());
certificates.add(entry.getCertificate());
}
return Collections.unmodifiableSet(identifiers);
return Collections.unmodifiableSet(certificates);
} catch (SQLException e) {
throw new IOException("Cannot query for subkey lookup entries.", e);
}
}
@Override
public void storeIdentifierForSubkeyId(long subkeyId, String identifier)
throws IOException {
public void storeCertificateSubkeyIds(String certificate, List<Long> subkeyIds) throws IOException {
try {
insert(subkeyId, identifier, true);
insertValues(certificate, subkeyIds);
} catch (SQLException e) {
throw new IOException("Cannot store subkey lookup entry in database.", e);
throw new IOException("Cannot store subkey lookup entries in database.", e);
}
}
}

2
pgp-cert-d-java-jdbc-sqlite-lookup/src/test/java/pgp/cert_d/jdbc/sqlite/EntryTest.java
View file

@ -16,6 +16,6 @@ public class EntryTest {
assertEquals(1, entry.getId());
assertEquals(123L, entry.getSubkeyId());
assertEquals("eb85bb5fa33a75e15e944e63f231550c4f47e38e", entry.getIdentifier());
assertEquals("eb85bb5fa33a75e15e944e63f231550c4f47e38e", entry.getCertificate());
}
}

28
pgp-cert-d-java-jdbc-sqlite-lookup/src/test/java/pgp/cert_d/jdbc/sqlite/SqliteSubkeyLookupTest.java
View file

@ -5,7 +5,6 @@
package pgp.cert_d.jdbc.sqlite;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.File;
@ -34,40 +33,35 @@ public class SqliteSubkeyLookupTest {
@Test
public void simpleInsertAndGet() throws IOException {
lookup.storeIdentifierForSubkeyId(123L, "eb85bb5fa33a75e15e944e63f231550c4f47e38e");
lookup.storeIdentifierForSubkeyId(234L, "eb85bb5fa33a75e15e944e63f231550c4f47e38e");
lookup.storeIdentifierForSubkeyId(234L, "d1a66e1a23b182c9980f788cfbfcc82a015e7330");
lookup.storeCertificateSubkeyIds("eb85bb5fa33a75e15e944e63f231550c4f47e38e", Arrays.asList(123L, 234L));
lookup.storeCertificateSubkeyIds("d1a66e1a23b182c9980f788cfbfcc82a015e7330", Collections.singletonList(234L));
assertEquals(Collections.singleton("eb85bb5fa33a75e15e944e63f231550c4f47e38e"), lookup.getIdentifiersForSubkeyId(123L));
assertEquals(Collections.singleton("eb85bb5fa33a75e15e944e63f231550c4f47e38e"), lookup.getCertificatesForSubkeyId(123L));
assertEquals(
new HashSet<>(Arrays.asList("eb85bb5fa33a75e15e944e63f231550c4f47e38e", "d1a66e1a23b182c9980f788cfbfcc82a015e7330")),
lookup.getIdentifiersForSubkeyId(234L));
lookup.getCertificatesForSubkeyId(234L));
}
@Test
public void getNonExistingSubkeyYieldsNull() throws IOException, SQLException {
assertTrue(lookup.get(6666666).isEmpty());
assertTrue(lookup.getIdentifiersForSubkeyId(6666666).isEmpty());
assertTrue(lookup.selectValues(6666666).isEmpty());
assertTrue(lookup.getCertificatesForSubkeyId(6666666).isEmpty());
}
@Test
public void secondInstanceLookupTest() throws IOException, SQLException {
lookup.storeIdentifierForSubkeyId(1337, "eb85bb5fa33a75e15e944e63f231550c4f47e38e");
assertEquals(Collections.singleton("eb85bb5fa33a75e15e944e63f231550c4f47e38e"), lookup.getIdentifiersForSubkeyId(1337));
lookup.storeCertificateSubkeyIds("eb85bb5fa33a75e15e944e63f231550c4f47e38e", Collections.singletonList(1337L));
assertEquals(Collections.singleton("eb85bb5fa33a75e15e944e63f231550c4f47e38e"), lookup.getCertificatesForSubkeyId(1337));
// do the lookup using a second db instance on the same file
SqliteSubkeyLookup secondInstance = SqliteSubkeyLookup.forDatabaseFile(databaseFile);
assertEquals(Collections.singleton("eb85bb5fa33a75e15e944e63f231550c4f47e38e"), secondInstance.getIdentifiersForSubkeyId(1337));
assertEquals(Collections.singleton("eb85bb5fa33a75e15e944e63f231550c4f47e38e"), secondInstance.getCertificatesForSubkeyId(1337));
}
@Test
public void ignoreInsertDuplicates() throws IOException {
lookup.storeIdentifierForSubkeyId(123L, "d1a66e1a23b182c9980f788cfbfcc82a015e7330");
lookup.storeCertificateSubkeyIds("d1a66e1a23b182c9980f788cfbfcc82a015e7330", Arrays.asList(123L, 234L));
// per default we ignore duplicates
lookup.storeIdentifierForSubkeyId(123L, "d1a66e1a23b182c9980f788cfbfcc82a015e7330");
// if we choose not to ignore duplicates, we raise an exception
assertThrows(SQLException.class, () ->
lookup.insert(123L, "d1a66e1a23b182c9980f788cfbfcc82a015e7330", false));
lookup.storeCertificateSubkeyIds("d1a66e1a23b182c9980f788cfbfcc82a015e7330", Arrays.asList(123L, 512L));
}
}

21
pgp-cert-d-java/src/main/java/pgp/cert_d/InMemorySubkeyLookup.java
View file

@ -4,20 +4,21 @@
package pgp.cert_d;
import pgp.certificate_store.SubkeyLookup;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import pgp.certificate_store.SubkeyLookup;
public class InMemorySubkeyLookup implements SubkeyLookup {
private static final Map<Long, Set<String>> subkeyMap = new HashMap<>();
@Override
public Set<String> getIdentifiersForSubkeyId(long subkeyId) {
public Set<String> getCertificatesForSubkeyId(long subkeyId) {
Set<String> identifiers = subkeyMap.get(subkeyId);
if (identifiers == null) {
return Collections.emptySet();
@ -26,13 +27,15 @@ public class InMemorySubkeyLookup implements SubkeyLookup {
}
@Override
public void storeIdentifierForSubkeyId(long subkeyId, String identifier) {
Set<String> identifiers = subkeyMap.get(subkeyId);
if (identifiers == null) {
identifiers = new HashSet<>();
subkeyMap.put(subkeyId, identifiers);
public void storeCertificateSubkeyIds(String certificate, List<Long> subkeyIds) {
for (long subkeyId : subkeyIds) {
Set<String> certificates = subkeyMap.get(subkeyId);
if (certificates == null) {
certificates = new HashSet<>();
subkeyMap.put(subkeyId, certificates);
}
certificates.add(certificate);
}
identifiers.add(identifier);
}
public void clear() {

28
pgp-cert-d-java/src/test/java/pgp/cert_d/SubkeyLookupTest.java
View file

@ -55,33 +55,33 @@ public class SubkeyLookupTest {
public void testInsertGet(SubkeyLookup subject) throws IOException {
// Initially all null
assertTrue(subject.getIdentifiersForSubkeyId(123).isEmpty());
assertTrue(subject.getIdentifiersForSubkeyId(1337).isEmpty());
assertTrue(subject.getIdentifiersForSubkeyId(420).isEmpty());
assertTrue(subject.getCertificatesForSubkeyId(123).isEmpty());
assertTrue(subject.getCertificatesForSubkeyId(1337).isEmpty());
assertTrue(subject.getCertificatesForSubkeyId(420).isEmpty());
// Store one val, others still null
subject.storeIdentifierForSubkeyId(123, "d1a66e1a23b182c9980f788cfbfcc82a015e7330");
subject.storeCertificateSubkeyIds("d1a66e1a23b182c9980f788cfbfcc82a015e7330", Collections.singletonList(123L));
assertEquals(Collections.singleton("d1a66e1a23b182c9980f788cfbfcc82a015e7330"), subject.getIdentifiersForSubkeyId(123));
assertTrue(subject.getIdentifiersForSubkeyId(1337).isEmpty());
assertTrue(subject.getIdentifiersForSubkeyId(420).isEmpty());
assertEquals(Collections.singleton("d1a66e1a23b182c9980f788cfbfcc82a015e7330"), subject.getCertificatesForSubkeyId(123));
assertTrue(subject.getCertificatesForSubkeyId(1337).isEmpty());
assertTrue(subject.getCertificatesForSubkeyId(420).isEmpty());
// Store other val, first stays intact
subject.storeIdentifierForSubkeyId(1337, "d1a66e1a23b182c9980f788cfbfcc82a015e7330");
subject.storeIdentifierForSubkeyId(420, "d1a66e1a23b182c9980f788cfbfcc82a015e7330");
subject.storeCertificateSubkeyIds("d1a66e1a23b182c9980f788cfbfcc82a015e7330", Collections.singletonList(1337L));
subject.storeCertificateSubkeyIds("d1a66e1a23b182c9980f788cfbfcc82a015e7330", Collections.singletonList(420L));
assertEquals(Collections.singleton("d1a66e1a23b182c9980f788cfbfcc82a015e7330"), subject.getIdentifiersForSubkeyId(123));
assertEquals(Collections.singleton("d1a66e1a23b182c9980f788cfbfcc82a015e7330"), subject.getIdentifiersForSubkeyId(1337));
assertEquals(Collections.singleton("d1a66e1a23b182c9980f788cfbfcc82a015e7330"), subject.getIdentifiersForSubkeyId(420));
assertEquals(Collections.singleton("d1a66e1a23b182c9980f788cfbfcc82a015e7330"), subject.getCertificatesForSubkeyId(123));
assertEquals(Collections.singleton("d1a66e1a23b182c9980f788cfbfcc82a015e7330"), subject.getCertificatesForSubkeyId(1337));
assertEquals(Collections.singleton("d1a66e1a23b182c9980f788cfbfcc82a015e7330"), subject.getCertificatesForSubkeyId(420));
// add additional entry for subkey
subject.storeIdentifierForSubkeyId(123, "eb85bb5fa33a75e15e944e63f231550c4f47e38e");
subject.storeCertificateSubkeyIds("eb85bb5fa33a75e15e944e63f231550c4f47e38e", Collections.singletonList(123L));
assertEquals(
new HashSet<>(Arrays.asList("eb85bb5fa33a75e15e944e63f231550c4f47e38e", "d1a66e1a23b182c9980f788cfbfcc82a015e7330")),
subject.getIdentifiersForSubkeyId(123));
subject.getCertificatesForSubkeyId(123));
}
}

13
pgp-certificate-store/src/main/java/pgp/certificate_store/SubkeyLookup.java
View file

@ -5,6 +5,7 @@
package pgp.certificate_store;
import java.io.IOException;
import java.util.List;
import java.util.Set;
public interface SubkeyLookup {
@ -16,13 +17,15 @@ public interface SubkeyLookup {
* @param subkeyId subkey id
* @return fingerprint of the certificate
*/
Set<String> getIdentifiersForSubkeyId(long subkeyId) throws IOException;
Set<String> getCertificatesForSubkeyId(long subkeyId) throws IOException;
/**
* Store a record of the subkey id that points to the fingerprint.
* Record, which certificate the subkey-ids in the list belong to.
* This method does not change the affiliation of subkey-ids not contained in the provided list.
*
* @param subkeyId subkey id
* @param identifier fingerprint of the certificate
* @param certificate certificate fingerprint
* @param subkeyIds subkey ids
* @throws IOException in case of an IO error
*/
void storeIdentifierForSubkeyId(long subkeyId, String identifier) throws IOException;
void storeCertificateSubkeyIds(String certificate, List<Long> subkeyIds) throws IOException;
}

14
pgpainless-cert-d/src/main/java/org/pgpainless/certificate_store/SharedPGPCertificateDirectoryAdapter.java
View file

@ -6,7 +6,9 @@ package org.pgpainless.certificate_store;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import pgp.cert_d.SharedPGPCertificateDirectory;
@ -106,18 +108,16 @@ public class SharedPGPCertificateDirectoryAdapter
return;
}
String fingerprint = certificate.getFingerprint();
for (Long subkeyId : certificate.getSubkeyIds()) {
storeIdentifierForSubkeyId(subkeyId, fingerprint);
}
storeCertificateSubkeyIds(fingerprint, new ArrayList<>(certificate.getSubkeyIds()));
}
@Override
public Set<String> getIdentifiersForSubkeyId(long subkeyId) throws IOException {
return subkeyLookup.getIdentifiersForSubkeyId(subkeyId);
public Set<String> getCertificatesForSubkeyId(long subkeyId) throws IOException {
return subkeyLookup.getCertificatesForSubkeyId(subkeyId);
}
@Override
public void storeIdentifierForSubkeyId(long subkeyId, String identifier) throws IOException {
subkeyLookup.storeIdentifierForSubkeyId(subkeyId, identifier);
public void storeCertificateSubkeyIds(String certificate, List<Long> subkeyIds) throws IOException {
subkeyLookup.storeCertificateSubkeyIds(certificate, subkeyIds);
}
}

4
pgpainless-cert-d/src/test/java/org/pgpainless/cert_d/SharedPGPCertificateDirectoryAdapterTest.java
View file

@ -84,7 +84,7 @@ public class SharedPGPCertificateDirectoryAdapterTest {
Set<Long> subkeys = certificate.getSubkeyIds();
assertEquals(expectedSubkeys, subkeys);
for (long subkey : subkeys) {
assertEquals(Collections.singleton(fingerprint), store.getIdentifiersForSubkeyId(subkey));
assertEquals(Collections.singleton(fingerprint), store.getCertificatesForSubkeyId(subkey));
}
Certificate retrieved = store.getCertificate(fingerprint);
@ -108,7 +108,7 @@ public class SharedPGPCertificateDirectoryAdapterTest {
Set<Long> subkeys = certificate.getSubkeyIds();
assertEquals(3, subkeys.size());
for (long subkey : subkeys) {
assertEquals(Collections.singleton(fingerprint), store.getIdentifiersForSubkeyId(subkey));
assertEquals(Collections.singleton(fingerprint), store.getCertificatesForSubkeyId(subkey));
}
Certificate retrieved = store.getCertificate(fingerprint);