ADDED: Database libraries MVP

Allows placing parts from an external database that reference symbols from another loaded library.

Includes:
- nanodbc wrapper
- database schematic library plugin
- basic tests

Fixes https://gitlab.com/kicad/code/kicad/-/issues/7436

cmake/KiCadQABuildUtils.cmake

@@ -30,8 +30,14 @@ option( KICAD_TEST_XML_OUTPUT
     "Cause unit tests to output xUnit results where possible for more granular CI reporting."
     OFF
 )
+
 mark_as_advanced( KICAD_TEST_XML_OUTPUT ) # Only CI tools need this
 
+option( KICAD_TEST_DATABASE_LIBRARIES
+    "Enable the database libraries QA tests (requires SQlite3 ODBC driver to be installed"
+    OFF
+)
+
 # This is a "meta" target that is used to collect all tests
 add_custom_target( qa_all_tests )
 

common/CMakeLists.txt

@@ -443,6 +443,9 @@ set( COMMON_SRCS
     project/project_archiver.cpp
     project/project_file.cpp
     project/project_local_settings.cpp
+
+    database/database_connection.cpp
+    database/database_lib_settings.cpp
     )
 
 add_library( common STATIC
@@ -492,6 +495,10 @@ target_include_directories( common PUBLIC
     $<TARGET_PROPERTY:pegtl,INTERFACE_INCLUDE_DIRECTORIES>
     )
 
+target_include_directories( common SYSTEM PUBLIC
+    $<TARGET_PROPERTY:nanodbc,INTERFACE_INCLUDE_DIRECTORIES>
+    )
+
 
 set( PCB_COMMON_SRCS
     base_screen.cpp

common/database/database_connection.cpp

@@ -0,0 +1,431 @@
+/*
+ * This program source code file is part of KiCad, a free EDA CAD application.
+ *
+ * Copyright (C) 2022 Jon Evans <jon@craftyjon.com>
+ * Copyright (C) 2022 KiCad Developers, see AUTHORS.txt for contributors.
+ *
+ * This program is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation, either version 3 of the License, or (at your
+ * option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <boost/locale.hpp>
+#include <fmt/core.h>
+#include <nanodbc/nanodbc.h>
+#include <sql.h> // SQL_IDENTIFIER_QUOTE_CHAR
+#include <wx/log.h>
+
+#include <database/database_connection.h>
+
+
+const char* const traceDatabase = "KICAD_DATABASE";
+
+/**
+ * When Unicode support is enabled in nanodbc, string formats are used matching the appropriate
+ * character set of the platform.  KiCad uses UTF-8 encoded strings internally, but different
+ * platforms use different encodings for SQL strings.  Unicode mode must be enabled for compilation
+ * on Windows, since Visual Studio forces the use of Unicode SQL headers if any part of the project
+ * has Unicode enabled.
+ */
+
+/**
+ * Converts a string from KiCad-native to nanodbc-native
+ * @param aString is a UTF-8 encoded string
+ * @return a string in nanodbc's platform-specific representation
+ */
+nanodbc::string fromUTF8( const std::string& aString )
+{
+    return boost::locale::conv::utf_to_utf<nanodbc::string::value_type>( aString );
+}
+
+
+/**
+ * Converts a string from nanodbc-native to KiCad-native
+ * @param aString is a string encoded in nanodbc's platform-specific way
+ * @return a string with UTF-8 encoding
+ */
+std::string toUTF8( const nanodbc::string& aString )
+{
+    return boost::locale::conv::utf_to_utf<char>( aString );
+}
+
+
+DATABASE_CONNECTION::DATABASE_CONNECTION( const std::string& aDataSourceName,
+                                          const std::string& aUsername,
+                                          const std::string& aPassword, int aTimeoutSeconds,
+                                          bool aConnectNow ) :
+        m_quoteChar( '"' )
+{
+    m_dsn     = aDataSourceName;
+    m_user    = aUsername;
+    m_pass    = aPassword;
+    m_timeout = aTimeoutSeconds;
+
+    if( aConnectNow )
+        Connect();
+}
+
+
+DATABASE_CONNECTION::DATABASE_CONNECTION( const std::string& aConnectionString,
+                                          int aTimeoutSeconds, bool aConnectNow ) :
+        m_quoteChar( '"' )
+{
+    m_connectionString = aConnectionString;
+    m_timeout          = aTimeoutSeconds;
+
+    if( aConnectNow )
+        Connect();
+}
+
+
+DATABASE_CONNECTION::~DATABASE_CONNECTION()
+{
+    Disconnect();
+    m_conn.reset();
+}
+
+
+bool DATABASE_CONNECTION::Connect()
+{
+    nanodbc::string dsn  = fromUTF8( m_dsn );
+    nanodbc::string user = fromUTF8( m_user );
+    nanodbc::string pass = fromUTF8( m_pass );
+    nanodbc::string cs   = fromUTF8( m_connectionString );
+
+    try
+    {
+        if( cs.empty() )
+        {
+            wxLogTrace( traceDatabase, wxT( "Creating connection to DSN %s" ), m_dsn );
+            m_conn = std::make_unique<nanodbc::connection>( dsn, user, pass, m_timeout );
+        }
+        else
+        {
+            wxLogTrace( traceDatabase, wxT( "Creating connection with connection string" ) );
+            m_conn = std::make_unique<nanodbc::connection>( cs, m_timeout );
+        }
+    }
+    catch( nanodbc::database_error& e )
+    {
+        m_lastError = e.what();
+        return false;
+    }
+
+    if( IsConnected() )
+    {
+        syncTables();
+        cacheColumns();
+        getQuoteChar();
+    }
+
+    return IsConnected();
+}
+
+
+bool DATABASE_CONNECTION::Disconnect()
+{
+    if( !m_conn )
+    {
+        wxLogTrace( traceDatabase, wxT( "Note: Disconnect() called without valid connection" ) );
+        return false;
+    }
+
+    m_conn->disconnect();
+
+    return !m_conn->connected();
+}
+
+
+bool DATABASE_CONNECTION::IsConnected() const
+{
+    if( !m_conn )
+        return false;
+
+    return m_conn->connected();
+}
+
+
+bool DATABASE_CONNECTION::syncTables()
+{
+    if( !m_conn )
+        return false;
+
+    m_tables.clear();
+
+    try
+    {
+        nanodbc::catalog catalog( *m_conn );
+        nanodbc::catalog::tables tables = catalog.find_tables();
+
+        while( tables.next() )
+        {
+            std::string key = toUTF8( tables.table_name() );
+            m_tables[key] = toUTF8( tables.table_type() );
+        }
+    }
+    catch( nanodbc::database_error& e )
+    {
+        m_lastError = e.what();
+        wxLogTrace( traceDatabase, wxT( "Exception while syncing tables: %s" ), m_lastError );
+        return false;
+    }
+
+    return true;
+}
+
+
+bool DATABASE_CONNECTION::cacheColumns()
+{
+    if( !m_conn )
+        return false;
+
+    m_columnCache.clear();
+
+    for( const auto& tableIter : m_tables )
+    {
+        try
+        {
+            nanodbc::catalog catalog( *m_conn );
+            nanodbc::catalog::columns columns =
+                    catalog.find_columns( NANODBC_TEXT( "" ), fromUTF8( tableIter.first ) );
+
+            while( columns.next() )
+            {
+                std::string columnKey = toUTF8( columns.column_name() );
+                m_columnCache[tableIter.first][columnKey] = columns.data_type();
+            }
+                
+        }
+        catch( nanodbc::database_error& e )
+        {
+            m_lastError = e.what();
+            wxLogTrace( traceDatabase, wxT( "Exception while syncing columns for table %s: %s" ),
+                        tableIter.first, m_lastError );
+            return false;
+        }
+    }
+
+    return true;
+}
+
+
+bool DATABASE_CONNECTION::getQuoteChar()
+{
+    if( !m_conn )
+        return false;
+
+    try
+    {
+        nanodbc::string qc = m_conn->get_info<nanodbc::string>( SQL_IDENTIFIER_QUOTE_CHAR );
+
+        if( qc.empty() )
+            return false;
+
+        m_quoteChar = *toUTF8( qc ).begin();
+
+        wxLogTrace( traceDatabase, wxT( "Quote char retrieved: %c" ), m_quoteChar );
+    }
+    catch( nanodbc::database_error& e )
+    {
+        wxLogTrace( traceDatabase, wxT( "Exception while querying quote char: %s" ), m_lastError );
+        return false;
+    }
+
+    return true;
+}
+
+
+bool DATABASE_CONNECTION::SelectOne( const std::string& aTable,
+                                     const std::pair<std::string, std::string>& aWhere,
+                                     DATABASE_CONNECTION::ROW& aResult )
+{
+    if( !m_conn )
+    {
+        wxLogTrace( traceDatabase, wxT( "Called SelectOne without valid connection!" ) );
+        return false;
+    }
+
+    auto tableMapIter = m_tables.find( aTable );
+
+    if( tableMapIter == m_tables.end() )
+    {
+        wxLogTrace( traceDatabase, wxT( "SelectOne: requested table %s not found in cache" ),
+                    aTable );
+        return false;
+    }
+
+    const std::string& tableName = tableMapIter->first;
+
+    if( !m_columnCache.count( tableName ) )
+    {
+        wxLogTrace( traceDatabase, wxT( "SelectOne: requested table %s missing from column cache" ),
+                    tableName );
+        return false;
+    }
+
+    auto columnCacheIter = m_columnCache.at( tableName ).find( aWhere.first );
+
+    if( columnCacheIter == m_columnCache.at( tableName ).end() )
+    {
+        wxLogTrace( traceDatabase, wxT( "SelectOne: requested column %s not found in cache for %s" ),
+                    aWhere.first, tableName );
+        return false;
+    }
+
+    const std::string& columnName = columnCacheIter->first;
+
+    nanodbc::statement statement( *m_conn );
+
+    nanodbc::string query = fromUTF8( fmt::format( "SELECT * FROM {}{}{} WHERE {}{}{} = ?",
+                                                   m_quoteChar, tableName, m_quoteChar,
+                                                   m_quoteChar, columnName, m_quoteChar ) );
+
+    try
+    {
+        statement.prepare( query );
+        statement.bind( 0, aWhere.second.c_str() );
+    }
+    catch( nanodbc::database_error& e )
+    {
+        m_lastError = e.what();
+        wxLogTrace( traceDatabase, wxT( "Exception while preparing statement for SelectOne: %s" ),
+                    m_lastError );
+        return false;
+    }
+
+    wxLogTrace( traceDatabase, wxT( "SelectOne: `%s` with parameter `%s`" ), toUTF8( query ),
+                aWhere.second );
+
+    nanodbc::result results;
+
+    try
+    {
+        results = nanodbc::execute( statement );
+    }
+    catch( nanodbc::database_error& e )
+    {
+        m_lastError = e.what();
+        wxLogTrace( traceDatabase, wxT( "Exception while executing statement for SelectOne: %s" ),
+                    m_lastError );
+        return false;
+    }
+
+    if( !results.first() )
+    {
+        wxLogTrace( traceDatabase, wxT( "SelectOne: no results returned from query" ) );
+        return false;
+    }
+
+    wxLogTrace( traceDatabase, wxT( "SelectOne: %ld results returned from query" ),
+                results.rows() );
+
+    aResult.clear();
+
+    try
+    {
+        for( short i = 0; i < results.columns(); ++i )
+        {
+            std::string column = toUTF8( results.column_name( i ) );
+            aResult[ column ] = toUTF8( results.get<nanodbc::string>( i, NANODBC_TEXT( "" ) ) );
+        }
+    }
+    catch( nanodbc::database_error& e )
+    {
+        m_lastError = e.what();
+        wxLogTrace( traceDatabase, wxT( "Exception while parsing results from SelectOne: %s" ),
+                    m_lastError );
+        return false;
+    }
+
+    return true;
+}
+
+
+bool DATABASE_CONNECTION::SelectAll( const std::string& aTable, std::vector<ROW>& aResults )
+{
+    if( !m_conn )
+    {
+        wxLogTrace( traceDatabase, wxT( "Called SelectAll without valid connection!" ) );
+        return false;
+    }
+
+    auto tableMapIter = m_tables.find( aTable );
+
+    if( tableMapIter == m_tables.end() )
+    {
+        wxLogTrace( traceDatabase, wxT( "SelectAll: requested table %s not found in cache" ),
+                    aTable );
+        return false;
+    }
+
+    nanodbc::statement statement( *m_conn );
+
+    nanodbc::string query = fromUTF8( fmt::format( "SELECT * FROM {}{}{}", m_quoteChar,
+                                                   tableMapIter->first, m_quoteChar ) );
+
+    wxLogTrace( traceDatabase, wxT( "SelectAll: `%s`" ), toUTF8( query ) );
+
+    try
+    {
+        statement.prepare( query );
+    }
+    catch( nanodbc::database_error& e )
+    {
+        m_lastError = e.what();
+        wxLogTrace( traceDatabase, wxT( "Exception while preparing query for SelectAll: %s" ),
+                    m_lastError );
+        return false;
+    }
+
+    nanodbc::result results;
+
+    try
+    {
+        results = nanodbc::execute( statement );
+    }
+    catch( nanodbc::database_error& e )
+    {
+        m_lastError = e.what();
+        wxLogTrace( traceDatabase, wxT( "Exception while executing query for SelectAll: %s" ),
+                    m_lastError );
+        return false;
+    }
+
+    if( !results.first() )
+        return false;
+
+    try
+    {
+        do
+        {
+            ROW result;
+
+            for( short j = 0; j < results.columns(); ++j )
+            {
+                std::string column = toUTF8( results.column_name( j ) );
+                result[column] = toUTF8( results.get<nanodbc::string>( j, NANODBC_TEXT( "" ) ) );
+            }
+
+            aResults.emplace_back( std::move( result ) );
+
+        } while( results.next() );
+    }
+    catch( nanodbc::database_error& e )
+    {
+        m_lastError = e.what();
+        wxLogTrace( traceDatabase, wxT( "Exception while parsing results from SelectAll: %s" ),
+                    m_lastError );
+        return false;
+    }
+
+    return true;
+}

common/database/database_lib_settings.cpp

@@ -0,0 +1,105 @@
+/*
+* This program source code file is part of KiCad, a free EDA CAD application.
+*
+* Copyright (C) 2022 Jon Evans <jon@craftyjon.com>
+* Copyright (C) 2022 KiCad Developers, see AUTHORS.TXT for contributors.
+*
+* This program is free software; you can redistribute it and/or
+* modify it under the terms of the GNU General Public License
+* as published by the Free Software Foundation; either version 2
+* of the License, or (at your option) any later version.
+*
+* This program is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+* GNU General Public License for more details.
+*
+* You should have received a copy of the GNU General Public License
+* along with this program; if not, you may find one here:
+* http://www.gnu.org/licenses/old-licenses/gpl-2.0.html
+* or you may search the http://www.gnu.org website for the version 2 license,
+* or you may write to the Free Software Foundation, Inc.,
+* 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
+*/
+
+#include <nlohmann/json.hpp>
+
+#include <database/database_lib_settings.h>
+#include <settings/parameters.h>
+#include <wildcards_and_files_ext.h>
+
+
+const int dblibSchemaVersion = 0;
+
+
+DATABASE_LIB_SETTINGS::DATABASE_LIB_SETTINGS( const std::string& aFilename ) :
+        JSON_SETTINGS( aFilename, SETTINGS_LOC::NONE, dblibSchemaVersion )
+{
+
+    m_params.emplace_back( new PARAM<std::string>( "source.dsn", &m_Source.dsn, "" ) );
+
+    m_params.emplace_back( new PARAM<std::string>( "source.username", &m_Source.username, "" ) );
+
+    m_params.emplace_back( new PARAM<std::string>( "source.password", &m_Source.password, "" ) );
+
+    m_params.emplace_back(
+            new PARAM<std::string>( "source.connection_string", &m_Source.connection_string, "" ) );
+
+    m_params.emplace_back( new PARAM<int>( "source.timeout_seconds", &m_Source.timeout, 2 ) );
+
+    m_params.emplace_back( new PARAM_LAMBDA<nlohmann::json>(
+            "libraries",
+            [&]() -> nlohmann::json
+            {
+                // TODO: implement this; libraries are read-only from KiCad at the moment
+                return {};
+            },
+            [&]( const nlohmann::json aObj )
+            {
+                m_Tables.clear();
+
+                if( !aObj.is_array() )
+                    return;
+
+                for( const nlohmann::json& entry : aObj )
+                {
+                    if( entry.empty() || !entry.is_object() )
+                        continue;
+
+                    DATABASE_LIB_TABLE table;
+
+                    table.name           = entry["name"].get<std::string>();
+                    table.table          = entry["table"].get<std::string>();
+                    table.key_col        = entry["key"].get<std::string>();
+                    table.symbols_col    = entry["symbols"].get<std::string>();
+                    table.footprints_col = entry["footprints"].get<std::string>();
+
+                    if( entry.contains( "fields" ) && entry["fields"].is_array() )
+                    {
+                        for( const nlohmann::json& fieldJson : entry["fields"] )
+                        {
+                            if( fieldJson.empty() || !fieldJson.is_object() )
+                                continue;
+
+                            table.fields.emplace_back(
+                                    DATABASE_FIELD_MAPPING(
+                                    {
+                                        fieldJson["column"].get<std::string>(),
+                                        fieldJson["name"].get<std::string>(),
+                                        fieldJson["visible_on_add"].get<bool>(),
+                                        fieldJson["visible_in_chooser"].get<bool>()
+                                    } ) );
+                        }
+                    }
+
+                    m_Tables.emplace_back( std::move( table ) );
+                }
+            },
+            {} ) );
+}
+
+
+wxString DATABASE_LIB_SETTINGS::getFileExt() const
+{
+    return DatabaseLibraryFileExtension;
+}

common/lib_table_base.cpp

@@ -184,7 +184,6 @@ LIB_TABLE_ROW* LIB_TABLE::findRow( const wxString& aNickName, bool aCheckIfEnabl
 
     do
     {
-        std::lock_guard<std::recursive_mutex> lock( cur->m_nickIndexMutex );
         cur->ensureIndex();
 
         for( const std::pair<const wxString, int>& entry : cur->nickIndex )
@@ -301,6 +300,8 @@ bool LIB_TABLE::InsertRow( LIB_TABLE_ROW* aRow, bool doReplace )
 
     INDEX_CITER it = nickIndex.find( aRow->GetNickName() );
 
+    aRow->SetParent( this );
+
     if( it == nickIndex.end() )
     {
         rows.push_back( aRow );

common/wildcards_and_files_ext.cpp

@@ -140,6 +140,7 @@ const std::string GerberJobFileExtension( "gbrjob" );
 const std::string HtmlFileExtension( "html" );
 const std::string EquFileExtension( "equ" );
 const std::string HotkeyFileExtension( "hotkeys" );
+const std::string DatabaseLibraryFileExtension( "kicad_dbl" );
 
 const std::string ArchiveFileExtension( "zip" );
 
@@ -206,10 +207,19 @@ wxString LegacySymbolLibFileWildcard()
 }
 
 
+wxString DatabaseLibFileWildcard()
+{
+    return _( "KiCad database library files" )
+           + AddFileExtListToFilter( { DatabaseLibraryFileExtension } );
+}
+
+
 wxString AllSymbolLibFilesWildcard()
 {
     return _( "All KiCad symbol library files" )
-            + AddFileExtListToFilter( { KiCadSymbolLibFileExtension, "lib" } );
+            + AddFileExtListToFilter( { KiCadSymbolLibFileExtension,
+                                        DatabaseLibraryFileExtension,
+                                        "lib" } );
 }
 
 

eeschema/CMakeLists.txt

@@ -274,6 +274,7 @@ set( EESCHEMA_SRCS
     sch_plugins/legacy/sch_legacy_lib_plugin_cache.cpp
     sch_plugins/legacy/sch_legacy_plugin.cpp
     sch_plugins/legacy/sch_legacy_plugin_helpers.cpp
+    sch_plugins/database/sch_database_plugin.cpp
 
     # Some simulation features must be built even if libngspice is not linked.
     sim/spice_settings.cpp

eeschema/dialogs/panel_sym_lib_table.cpp

@@ -214,6 +214,7 @@ PANEL_SYM_LIB_TABLE::PANEL_SYM_LIB_TABLE( DIALOG_EDIT_LIBRARY_TABLES* aParent, P
 
     pluginChoices.Add( SCH_IO_MGR::ShowType( SCH_IO_MGR::SCH_KICAD ) );
     pluginChoices.Add( SCH_IO_MGR::ShowType( SCH_IO_MGR::SCH_LEGACY ) );
+    pluginChoices.Add( SCH_IO_MGR::ShowType( SCH_IO_MGR::SCH_DATABASE ) );
 
     EESCHEMA_SETTINGS* cfg = Pgm().GetSettingsManager().GetAppSettings<EESCHEMA_SETTINGS>();
 
@@ -435,6 +436,9 @@ bool PANEL_SYM_LIB_TABLE::verifyTables()
         {
             SYMBOL_LIB_TABLE_ROW& row = dynamic_cast<SYMBOL_LIB_TABLE_ROW&>( table->At( r ) );
 
+            if( !row.GetParent() )
+                row.SetParent( table );
+
             if( !row.GetIsEnabled() )
                 continue;
 
@@ -476,7 +480,8 @@ void PANEL_SYM_LIB_TABLE::browseLibrariesHandler( wxCommandEvent& event )
 {
     wxString wildcards = AllSymbolLibFilesWildcard()
                             + "|" + KiCadSymbolLibFileWildcard()
-                            + "|" + LegacySymbolLibFileWildcard();
+                            + "|" + LegacySymbolLibFileWildcard()
+                            + "|" + DatabaseLibFileWildcard();
 
     EESCHEMA_SETTINGS* cfg = Pgm().GetSettingsManager().GetAppSettings<EESCHEMA_SETTINGS>();
 

eeschema/lib_field.cpp

@@ -71,6 +71,7 @@ LIB_FIELD& LIB_FIELD::operator=( const LIB_FIELD& field )
     m_id = field.m_id;
     m_name = field.m_name;
     m_parent = field.m_parent;
+    m_autoAdded = field.m_autoAdded;
 
     SetText( field.GetText() );
     SetAttributes( field );
@@ -95,6 +96,8 @@ void LIB_FIELD::Init( int aId )
     // template fieldsnames' initial visibility is controlled by the template fieldname config.
     if( aId == DATASHEET_FIELD || aId == FOOTPRINT_FIELD )
         SetVisible( false );
+
+    m_autoAdded = false;
 }
 
 
@@ -191,6 +194,7 @@ void LIB_FIELD::Copy( LIB_FIELD* aTarget ) const
     aTarget->CopyText( *this );
     aTarget->SetAttributes( *this );
     aTarget->SetParent( m_parent );
+    aTarget->SetAutoAdded( IsAutoAdded() );
 }
 
 

eeschema/lib_field.h

@@ -173,6 +173,9 @@ public:
 
     bool IsMandatory() const;
 
+    bool IsAutoAdded() const { return m_autoAdded; }
+    void SetAutoAdded( bool aAutoAdded ) { m_autoAdded = aAutoAdded; }
+
 private:
 
     /**
@@ -209,6 +212,7 @@ private:
 
     int      m_id;         ///< @see enum MANDATORY_FIELD_T
     wxString m_name;       ///< Name (not the field text value itself, that is #EDA_TEXT::m_Text)
+    bool     m_autoAdded;  ///< Was this field automatically added to a LIB_SYMBOL?
 };
 
 #endif  //  CLASS_LIBENTRY_FIELDS_H

eeschema/sch_io_mgr.cpp

@@ -30,6 +30,7 @@
 
 #include <sch_plugins/altium/sch_altium_plugin.h>
 #include <sch_plugins/cadstar/cadstar_sch_archive_plugin.h>
+#include <sch_plugins/database/sch_database_plugin.h>
 #include <wildcards_and_files_ext.h>
 
 #define FMT_UNIMPLEMENTED   _( "Plugin \"%s\" does not implement the \"%s\" function." )
@@ -61,6 +62,7 @@ SCH_PLUGIN* SCH_IO_MGR::FindPlugin( SCH_FILE_T aFileType )
     case SCH_ALTIUM:          return new SCH_ALTIUM_PLUGIN();
     case SCH_CADSTAR_ARCHIVE: return new CADSTAR_SCH_ARCHIVE_PLUGIN();
     case SCH_EAGLE:           return new SCH_EAGLE_PLUGIN();
+    case SCH_DATABASE:        return new SCH_DATABASE_PLUGIN();
     default:                  return nullptr;
     }
 }
@@ -89,6 +91,7 @@ const wxString SCH_IO_MGR::ShowType( SCH_FILE_T aType )
     case SCH_ALTIUM:          return wxString( wxT( "Altium" ) );
     case SCH_CADSTAR_ARCHIVE: return wxString( wxT( "CADSTAR Schematic Archive" ) );
     case SCH_EAGLE:           return wxString( wxT( "EAGLE" ) );
+    case SCH_DATABASE:        return wxString( wxT( "Database" ) );
     default:                  return wxString::Format( _( "Unknown SCH_FILE_T value: %d" ),
                                                        aType );
     }
@@ -111,6 +114,8 @@ SCH_IO_MGR::SCH_FILE_T SCH_IO_MGR::EnumFromStr( const wxString& aType )
         return SCH_CADSTAR_ARCHIVE;
     else if( aType == wxT( "EAGLE" ) )
         return SCH_EAGLE;
+    else if( aType == wxT( "Database" ) )
+        return SCH_DATABASE;
 
     // wxASSERT( blow up here )
 

eeschema/sch_io_mgr.h

@@ -34,6 +34,7 @@ class SCH_SHEET;
 class SCH_SCREEN;
 class SCH_PLUGIN;
 class SCHEMATIC;
+class SYMBOL_LIB_TABLE;
 class KIWAY;
 class LIB_SYMBOL;
 class SYMBOL_LIB;
@@ -59,6 +60,7 @@ public:
                     SCH_ALTIUM,  ///< Altium file format
                     SCH_CADSTAR_ARCHIVE, ///< CADSTAR Schematic Archive
                     SCH_EAGLE,  ///< Autodesk Eagle file format
+                    SCH_DATABASE, ///< KiCad database library
 
                     // Add your schematic type here.
                     SCH_FILE_UNKNOWN
@@ -459,6 +461,12 @@ public:
      */
     virtual const wxString& GetError() const;
 
+    /**
+     * Some library plugins need to have access to their parent library table.
+     * @param aTable is the table this plugin is registered within.
+     */
+    virtual void SetLibTable( SYMBOL_LIB_TABLE* aTable ) {}
+
     //-----</PUBLIC SCH_PLUGIN API>------------------------------------------------
 
 

eeschema/sch_plugins/database/sch_database_plugin.cpp

@@ -0,0 +1,323 @@
+/*
+ * This program source code file is part of KiCad, a free EDA CAD application.
+ *
+ * Copyright (C) 2022 Jon Evans <jon@craftyjon.com>
+ * Copyright (C) 2022 KiCad Developers, see AUTHORS.txt for contributors.
+ *
+ * This program is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation, either version 3 of the License, or (at your
+ * option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <iostream>
+
+#include <boost/algorithm/string.hpp>
+
+#include <database/database_connection.h>
+#include <database/database_lib_settings.h>
+#include <fmt.h>
+#include <lib_symbol.h>
+#include <symbol_lib_table.h>
+
+#include "sch_database_plugin.h"
+
+
+SCH_DATABASE_PLUGIN::SCH_DATABASE_PLUGIN() :
+        m_libTable( nullptr ),
+        m_settings(),
+        m_conn()
+{
+}
+
+
+SCH_DATABASE_PLUGIN::~SCH_DATABASE_PLUGIN()
+{
+}
+
+
+void SCH_DATABASE_PLUGIN::EnumerateSymbolLib( wxArrayString&    aSymbolNameList,
+                                              const wxString&   aLibraryPath,
+                                              const PROPERTIES* aProperties )
+{
+    std::vector<LIB_SYMBOL*> symbols;
+    EnumerateSymbolLib( symbols, aLibraryPath, aProperties );
+
+    for( LIB_SYMBOL* symbol : symbols )
+        aSymbolNameList.Add( symbol->GetName() );
+}
+
+
+void SCH_DATABASE_PLUGIN::EnumerateSymbolLib( std::vector<LIB_SYMBOL*>& aSymbolList,
+                                              const wxString&           aLibraryPath,
+                                              const PROPERTIES*         aProperties )
+{
+    wxCHECK_RET( m_libTable, "Database plugin missing library table handle!" );
+    ensureSettings( aLibraryPath );
+    ensureConnection();
+
+    bool powerSymbolsOnly = ( aProperties &&
+                              aProperties->find( SYMBOL_LIB_TABLE::PropPowerSymsOnly ) !=
+                              aProperties->end() );
+
+    for( const DATABASE_LIB_TABLE& table : m_settings->m_Tables )
+    {
+        std::vector<DATABASE_CONNECTION::ROW> results;
+
+        if( !m_conn->SelectAll( table.table, results ) )
+        {
+            if( !m_conn->GetLastError().empty() )
+            {
+                wxString msg = wxString::Format( _( "Error reading database table %s: %s" ),
+                                                 table.table, m_conn->GetLastError() );
+                THROW_IO_ERROR( msg );
+            }
+
+            continue;
+        }
+
+        for( DATABASE_CONNECTION::ROW& result : results )
+        {
+            if( !result.count( table.key_col ) )
+                continue;
+
+            wxString name( fmt::format( "{}/{}", table.name,
+                                        std::any_cast<std::string>( result[table.key_col] ) ) );
+
+            LIB_SYMBOL* symbol = loadSymbolFromRow( name, table, result );
+
+            if( symbol && ( !powerSymbolsOnly || symbol->IsPower() ) )
+                aSymbolList.emplace_back( symbol );
+        }
+    }
+}
+
+
+LIB_SYMBOL* SCH_DATABASE_PLUGIN::LoadSymbol( const wxString&   aLibraryPath,
+                                             const wxString&   aAliasName,
+                                             const PROPERTIES* aProperties )
+{
+    wxCHECK( m_libTable, nullptr );
+    ensureSettings( aLibraryPath );
+    ensureConnection();
+
+    const DATABASE_LIB_TABLE* table = nullptr;
+
+    std::string tableName( aAliasName.BeforeFirst( '/' ).ToUTF8() );
+    std::string symbolName( aAliasName.AfterFirst( '/' ).ToUTF8() );
+
+    for( const DATABASE_LIB_TABLE& tableIter : m_settings->m_Tables )
+    {
+        if( tableIter.table == tableName )
+        {
+            table = &tableIter;
+            break;
+        }
+    }
+
+    if( !table )
+    {
+        wxLogTrace( traceDatabase, wxT( "LoadSymbol: table %s not found in config" ), tableName );
+        return nullptr;
+    }
+
+    DATABASE_CONNECTION::ROW result;
+
+    if( !m_conn->SelectOne( tableName, std::make_pair( table->key_col, symbolName ), result ) )
+    {
+        wxLogTrace( traceDatabase, wxT( "LoadSymbol: SelectOne (%s, %s) failed" ), table->key_col,
+                    symbolName );
+        return nullptr;
+    }
+
+    return loadSymbolFromRow( symbolName, *table, result );
+}
+
+
+bool SCH_DATABASE_PLUGIN::CheckHeader( const wxString& aFileName )
+{
+    // TODO: Implement this sometime; but CheckHeader isn't even called...
+    return true;
+}
+
+
+void SCH_DATABASE_PLUGIN::ensureSettings( const wxString& aSettingsPath )
+{
+    if( !m_settings )
+    {
+        std::string path( aSettingsPath.ToUTF8() );
+        m_settings = std::make_unique<DATABASE_LIB_SETTINGS>( path );
+        m_settings->SetReadOnly( true );
+
+        if( !m_settings->LoadFromFile() )
+        {
+            wxString msg = wxString::Format(
+                    _( "Could not load database library: settings file %s missing or invalid" ),
+                    aSettingsPath );
+
+            THROW_IO_ERROR( msg );
+        }
+    }
+    else
+    {
+        wxASSERT_MSG( aSettingsPath == m_settings->GetFilename(),
+                      "Path changed for database library without re-initializing plugin!" );
+    }
+}
+
+
+void SCH_DATABASE_PLUGIN::ensureConnection()
+{
+    wxCHECK_RET( m_settings, "Call ensureSettings before ensureConnection!" );
+
+    if( !m_conn )
+    {
+        if( m_settings->m_Source.connection_string.empty() )
+        {
+            m_conn = std::make_unique<DATABASE_CONNECTION>( m_settings->m_Source.dsn,
+                                                            m_settings->m_Source.username,
+                                                            m_settings->m_Source.password,
+                                                            m_settings->m_Source.timeout );
+        }
+        else
+        {
+            std::string cs = m_settings->m_Source.connection_string;
+            std::string basePath( wxFileName( m_settings->GetFilename() ).GetPath().ToUTF8() );
+
+            // Database drivers that use files operate on absolute paths, so provide a mechanism
+            // for specifing on-disk databases that live next to the kicad_dbl file
+            boost::replace_all( cs, "${CWD}", basePath );
+
+            m_conn = std::make_unique<DATABASE_CONNECTION>( cs, m_settings->m_Source.timeout );
+        }
+
+        if( !m_conn->IsConnected() )
+        {
+            wxString msg = wxString::Format(
+                    _( "Could not load database library: could not connect to database %s (%s)" ),
+                    m_settings->m_Source.dsn,
+                    m_conn->GetLastError() );
+
+            THROW_IO_ERROR( msg );
+        }
+    }
+}
+
+
+LIB_SYMBOL* SCH_DATABASE_PLUGIN::loadSymbolFromRow( const wxString& aSymbolName,
+                                                    const DATABASE_LIB_TABLE& aTable,
+                                                    const DATABASE_CONNECTION::ROW& aRow )
+{
+    LIB_SYMBOL* symbol = nullptr;
+
+    if( aRow.count( aTable.symbols_col ) )
+    {
+        // TODO: Support multiple options for symbol
+        std::string symbolIdStr = std::any_cast<std::string>( aRow.at( aTable.symbols_col ) );
+        LIB_ID symbolId;
+        symbolId.Parse( std::any_cast<std::string>( aRow.at( aTable.symbols_col ) ) );
+
+        LIB_SYMBOL* originalSymbol = m_libTable->LoadSymbol( symbolId );
+
+        if( originalSymbol )
+        {
+            wxLogTrace( traceDatabase, wxT( "loadSymbolFromRow: found original symbol '%s'" ),
+                        symbolIdStr );
+            symbol = originalSymbol->Duplicate();
+        }
+        else
+        {
+            wxLogTrace( traceDatabase, wxT( "loadSymboFromRow: source symbol '%s' not found, "
+                                            "will create empty symbol" ), symbolIdStr );
+        }
+    }
+
+    if( !symbol )
+    {
+        // Actual symbol not found: return metadata only; error will be indicated in the
+        // symbol chooser
+        symbol = new LIB_SYMBOL( aSymbolName );
+    }
+    else
+    {
+        symbol->SetName( aSymbolName );
+    }
+
+    if( aRow.count( aTable.footprints_col ) )
+    {
+        // TODO: Support multiple footprint choices
+        std::string footprints = std::any_cast<std::string>( aRow.at( aTable.footprints_col ) );
+        wxString footprint = wxString( footprints.c_str(), wxConvUTF8 ).BeforeFirst( ';' );
+        symbol->GetFootprintField().SetText( footprint );
+    }
+    else
+    {
+        wxLogTrace( traceDatabase, wxT( "loadSymboFromRow: footprint field %s not found." ),
+                    aTable.footprints_col );
+    }
+
+    for( const DATABASE_FIELD_MAPPING& mapping : aTable.fields )
+    {
+        if( !aRow.count( mapping.column ) )
+        {
+            wxLogTrace( traceDatabase, wxT( "loadSymboFromRow: field %s not found in result" ),
+                        mapping.column );
+            continue;
+        }
+
+        wxString value( std::any_cast<std::string>( aRow.at( mapping.column ) ).c_str(),
+                        wxConvUTF8 );
+
+        if( mapping.name == wxT( "ki_description" ) )
+        {
+            symbol->SetDescription( value );
+            continue;
+        }
+        else if( mapping.name == wxT( "ki_keywords" ) )
+        {
+            symbol->SetKeyWords( value );
+            continue;
+        }
+        else if( mapping.name == wxT( "ki_fp_filters" ) )
+        {
+            // TODO: Handle this here?
+            continue;
+        }
+        else if( mapping.name == wxT( "Value" ) )
+        {
+            LIB_FIELD& field = symbol->GetValueField();
+            field.SetText( value );
+            field.SetVisible( mapping.visible_on_add );
+            continue;
+        }
+        else if( mapping.name == wxT( "Datasheet" ) )
+        {
+            LIB_FIELD& field = symbol->GetDatasheetField();
+            field.SetText( value );
+            field.SetVisible( mapping.visible_on_add );
+
+            if( mapping.visible_on_add )
+                field.SetAutoAdded( true );
+
+            continue;
+        }
+
+        LIB_FIELD* field = new LIB_FIELD( symbol->GetNextAvailableFieldId() );
+        field->SetName( mapping.name );
+        field->SetText( value );
+        field->SetVisible( mapping.visible_on_add );
+        field->SetAutoAdded( true );
+
+        symbol->AddField( field );
+    }
+
+    return symbol;
+}

eeschema/sch_plugins/database/sch_database_plugin.h

@@ -0,0 +1,106 @@
+/*
+ * This program source code file is part of KiCad, a free EDA CAD application.
+ *
+ * Copyright (C) 2022 Jon Evans <jon@craftyjon.com>
+ * Copyright (C) 2022 KiCad Developers, see AUTHORS.txt for contributors.
+ *
+ * This program is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation, either version 3 of the License, or (at your
+ * option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef KICAD_SCH_DATABASE_PLUGIN_H
+#define KICAD_SCH_DATABASE_PLUGIN_H
+
+#include <database/database_connection.h>
+#include <sch_io_mgr.h>
+#include <wildcards_and_files_ext.h>
+
+
+class DATABASE_LIB_SETTINGS;
+struct DATABASE_LIB_TABLE;
+
+
+/**
+ * A KiCad database library provides both symbol and footprint metadata, so there are "shim" plugins
+ * on both the symbol and footprint side of things that expose the database contents to the
+ * schematic and board editors.  The architecture of these is slightly different from the other
+ * plugins because the backing file is just a configuration file rather than something that
+ * contains symbol or footprint data.
+ */
+class SCH_DATABASE_PLUGIN : public SCH_PLUGIN
+{
+public:
+
+    SCH_DATABASE_PLUGIN();
+    virtual ~SCH_DATABASE_PLUGIN();
+
+    const wxString GetName() const override
+    {
+        return wxT( "Database library" );
+    }
+
+    const wxString GetLibraryFileExtension() const override
+    {
+        return DatabaseLibraryFileExtension;
+    }
+
+    const wxString GetFileExtension() const override
+    {
+        wxFAIL_MSG( "Database libraries are not schematic files!  Fix call site." );
+        return DatabaseLibraryFileExtension;
+    }
+
+    int GetModifyHash() const override { return 0; }
+
+    void EnumerateSymbolLib( wxArrayString&    aSymbolNameList,
+                             const wxString&   aLibraryPath,
+                             const PROPERTIES* aProperties = nullptr ) override;
+
+    void EnumerateSymbolLib( std::vector<LIB_SYMBOL*>& aSymbolList,
+                             const wxString&           aLibraryPath,
+                             const PROPERTIES*         aProperties = nullptr ) override;
+
+    LIB_SYMBOL* LoadSymbol( const wxString& aLibraryPath, const wxString& aAliasName,
+                            const PROPERTIES* aProperties = nullptr ) override;
+
+    bool CheckHeader( const wxString& aFileName ) override;
+
+    // Database libraries can never be written using the symbol editing API
+    bool IsSymbolLibWritable( const wxString& aLibraryPath ) override
+    {
+        return false;
+    }
+
+    void SetLibTable( SYMBOL_LIB_TABLE* aTable ) override
+    {
+        m_libTable = aTable;
+    }
+
+private:
+    void ensureSettings( const wxString& aSettingsPath );
+
+    void ensureConnection();
+
+    LIB_SYMBOL* loadSymbolFromRow( const wxString& aSymbolName,
+                                   const DATABASE_LIB_TABLE& aTable,
+                                   const DATABASE_CONNECTION::ROW& aRow );
+
+    SYMBOL_LIB_TABLE* m_libTable;
+
+    std::unique_ptr<DATABASE_LIB_SETTINGS> m_settings;
+
+    std::unique_ptr<DATABASE_CONNECTION> m_conn;
+
+};
+
+#endif //KICAD_SCH_DATABASE_PLUGIN_H

eeschema/sch_symbol.cpp

@@ -1018,7 +1018,8 @@ bool SCH_SYMBOL::ResolveTextVar( wxString* token, int aDepth ) const
     SCHEMATIC* schematic = Schematic();
 
     // SCH_SYMOL object has no context outside a schematic.
-    wxCHECK( schematic, false );
+    if( !schematic )
+        return false;
 
     for( int i = 0; i < MANDATORY_FIELDS; ++i )
     {

eeschema/symbol_lib_table.cpp

@@ -76,6 +76,7 @@ bool SYMBOL_LIB_TABLE_ROW::Refresh()
 
         plugin.set( SCH_IO_MGR::FindPlugin( type ) );
         SetLoaded( false );
+        plugin->SetLibTable( static_cast<SYMBOL_LIB_TABLE*>( GetParent() ) );
         plugin->EnumerateSymbolLib( dummyList, GetFullURI( true ), GetProperties() );
         SetLoaded( true );
         return true;
@@ -312,7 +313,10 @@ SYMBOL_LIB_TABLE_ROW* SYMBOL_LIB_TABLE::FindRow( const wxString& aNickname, bool
     // instantiate a PLUGIN of the proper kind if it is not already in this
     // SYMBOL_LIB_TABLE_ROW.
     if( !row->plugin )
+    {
         row->setPlugin( SCH_IO_MGR::FindPlugin( row->type ) );
+        row->plugin->SetLibTable( this );
+    }
 
     return row;
 }
@@ -327,7 +331,7 @@ void SYMBOL_LIB_TABLE::LoadSymbolLib( std::vector<LIB_SYMBOL*>& aSymbolList,
     wxString options = row->GetOptions();
 
     if( aPowerSymbolsOnly )
-        row->SetOptions( row->GetOptions() + " " + PropPowerSymsOnly );
+            row->SetOptions( row->GetOptions() + " " + PropPowerSymsOnly );
 
     row->SetLoaded( false );
     row->plugin->EnumerateSymbolLib( aSymbolList, row->GetFullURI( true ), row->GetProperties() );
@@ -354,7 +358,7 @@ LIB_SYMBOL* SYMBOL_LIB_TABLE::LoadSymbol( const wxString& aNickname, const wxStr
 {
     SYMBOL_LIB_TABLE_ROW* row = FindRow( aNickname, true );
 
-    if( !row || !row->plugin )
+    if( !row || !row->plugin || !row->GetIsLoaded() )
         return nullptr;
 
     LIB_SYMBOL* symbol = row->plugin->LoadSymbol( row->GetFullURI( true ), aSymbolName,

eeschema/tools/sch_drawing_tools.cpp

@@ -328,6 +328,9 @@ int SCH_DRAWING_TOOLS::PlaceSymbol( const TOOL_EVENT& aEvent )
                 addSymbol( symbol );
                 annotate();
 
+                if( m_frame->eeconfig()->m_AutoplaceFields.enable )
+                    symbol->AutoplaceFields( /* aScreen */ nullptr, /* aManual */ false );
+
                 // Update cursor now that we have a symbol
                 setCursor();
             }

eeschema/widgets/symbol_preview_widget.cpp

@@ -196,6 +196,18 @@ void SYMBOL_PREVIEW_WIDGET::DisplaySymbol( const LIB_ID& aSymbolID, int aUnit, i
         // This will flatten derived parts so that the correct final symbol can be shown.
         m_previewItem = symbol.release();
 
+        // Hide fields that were added automatically by the library (for example, when using
+        // database libraries) as they don't have a valid position yet, and we don't support
+        // autoplacing fields on library symbols yet.
+        std::vector<LIB_FIELD*> previewFields;
+        m_previewItem->GetFields( previewFields );
+
+        for( LIB_FIELD* field : previewFields )
+        {
+            if( field->IsAutoAdded() )
+                field->SetVisible( false );
+        }
+
         // If unit isn't specified for a multi-unit part, pick the first.  (Otherwise we'll
         // draw all of them.)
         settings->m_ShowUnit = ( m_previewItem->IsMulti() && aUnit == 0 ) ? 1 : aUnit;

include/database/database_connection.h

@@ -0,0 +1,109 @@
+/*
+ * This program source code file is part of KiCad, a free EDA CAD application.
+ *
+ * Copyright (C) 2022 Jon Evans <jon@craftyjon.com>
+ * Copyright (C) 2022 KiCad Developers, see AUTHORS.txt for contributors.
+ *
+ * This program is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation, either version 3 of the License, or (at your
+ * option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef KICAD_DATABASE_CONNECTION_H
+#define KICAD_DATABASE_CONNECTION_H
+
+#include <any>
+#include <map>
+#include <memory>
+#include <vector>
+
+extern const char* const traceDatabase;
+
+
+namespace nanodbc
+{
+    class connection;
+}
+
+
+class DATABASE_CONNECTION
+{
+public:
+    static const long DEFAULT_TIMEOUT = 10;
+
+    typedef std::map<std::string, std::any> ROW;
+
+    DATABASE_CONNECTION( const std::string& aDataSourceName, const std::string& aUsername,
+                         const std::string& aPassword, int aTimeoutSeconds = DEFAULT_TIMEOUT,
+                         bool aConnectNow = true );
+
+    DATABASE_CONNECTION( const std::string& aConnectionString,
+                         int aTimeoutSeconds = DEFAULT_TIMEOUT,
+                         bool aConnectNow = true );
+
+    ~DATABASE_CONNECTION();
+
+    bool Connect();
+
+    bool Disconnect();
+
+    bool IsConnected() const;
+
+    /**
+     * Retrieves a single row from a database table.  Table and column names are cached when the
+     * connection is created, so schema changes to the database won't be recognized unless the
+     * connection is recreated.
+     * @param aTable the name of a table in the database
+     * @param aWhere column to search, and the value to search for
+     * @param aResult will be filled with a row in the database if one was found
+     * @return true if aResult was filled; false otherwise
+     */
+    bool SelectOne( const std::string& aTable, const std::pair<std::string, std::string>& aWhere,
+                    ROW& aResult );
+
+    /**
+     * Retrieves all rows from a database table.
+     * @param aTable the name of a table in the database
+     * @param aResults will be filled with all rows in the table
+     * @return true if the query succeeded and at least one ROW was found, false otherwise
+     */
+    bool SelectAll( const std::string& aTable, std::vector<ROW>& aResults );
+
+    std::string GetLastError() const { return m_lastError; }
+
+private:
+    bool syncTables();
+
+    bool cacheColumns();
+
+    bool getQuoteChar();
+
+    std::unique_ptr<nanodbc::connection> m_conn;
+
+    std::string m_dsn;
+    std::string m_user;
+    std::string m_pass;
+    std::string m_connectionString;
+
+    std::string m_lastError;
+
+    std::map<std::string, std::string> m_tables;
+
+    /// Map of table -> map of column name -> data type
+    std::map<std::string, std::map<std::string, int>> m_columnCache;
+
+    long m_timeout;
+
+    char m_quoteChar;
+};
+
+#endif //KICAD_DATABASE_CONNECTION_H

include/database/database_lib_settings.h

@@ -0,0 +1,94 @@
+/*
+ * This program source code file is part of KiCad, a free EDA CAD application.
+ *
+ * Copyright (C) 2022 Jon Evans <jon@craftyjon.com>
+ * Copyright (C) 2022 KiCad Developers, see AUTHORS.txt for contributors.
+ *
+ * This program is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation, either version 3 of the License, or (at your
+ * option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef KICAD_DATABASE_LIB_SETTINGS_H
+#define KICAD_DATABASE_LIB_SETTINGS_H
+
+#include <settings/json_settings.h>
+
+
+enum class DATABASE_SOURCE_TYPE
+{
+    ODBC,
+    INVALID
+};
+
+
+struct DATABASE_SOURCE
+{
+    DATABASE_SOURCE_TYPE type;
+    std::string          dsn;
+    std::string          username;
+    std::string          password;
+    int                  timeout;
+    std::string          connection_string;
+};
+
+
+struct DATABASE_FIELD_MAPPING
+{
+    std::string column;              ///< Database column name
+    std::string name;                ///< KiCad field name
+    bool        visible_on_add;      ///< Whether to show the field when placing the symbol
+    bool        visible_in_chooser;  ///< Whether the column is shown by default in the chooser
+};
+
+
+/**
+ * A database library table will be mapped to a sub-library provided by the database library entry
+ * in the KiCad symbol/footprint library table.  A single database library config file (managed by
+ * this class) may contain more than one table mapping, and each will have its own nickname.
+ *
+ * The LIB_ID for parts placed from this library will be constructed from the nickname of the
+ * database library itself, plus the nickname of the particular sub-library and the value of the
+ * key column for the placed part.
+ *
+ * For example, if a database library is configured with the nickname "PartsDB" and it provides a
+ * table called "Capacitors", with `key_col` set to "Part Number", the LIB_ID for a capacitor placed
+ * from this table will look something like `PartsDB-Capacitors:CAP-001`
+ */
+struct DATABASE_LIB_TABLE
+{
+    std::string name;              ///< KiCad library nickname (will form part of the LIB_ID)
+    std::string table;             ///< Database table to pull content from
+    std::string key_col;           ///< Unique key column name (will form part of the LIB_ID)
+    std::string symbols_col;       ///< Column name containing KiCad symbol refs
+    std::string footprints_col;    ///< Column name containing KiCad footprint refs
+
+    std::vector<DATABASE_FIELD_MAPPING> fields;
+};
+
+
+class DATABASE_LIB_SETTINGS : public JSON_SETTINGS
+{
+public:
+    DATABASE_LIB_SETTINGS( const std::string& aFilename );
+
+    virtual ~DATABASE_LIB_SETTINGS() {}
+
+    DATABASE_SOURCE m_Source;
+
+    std::vector<DATABASE_LIB_TABLE> m_Tables;
+
+protected:
+    wxString getFileExt() const override;
+};
+
+#endif //KICAD_DATABASE_LIB_SETTINGS_H

include/fmt.h

@@ -0,0 +1,34 @@
+/*
+ * This program source code file is part of KiCad, a free EDA CAD application.
+ *
+ * Copyright (C) 2022 Jon Evans <jon@craftyjon.com>
+ * Copyright (C) 2022 KiCad Developers, see AUTHORS.txt for contributors.
+ *
+ * This program is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation, either version 3 of the License, or (at your
+ * option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef KICAD_FMT_H
+#define KICAD_FMT_H
+
+#include <fmt/core.h>
+#include <fmt/ostream.h>
+#include <wx/wx.h>
+
+// {fmt} formatter for wxString
+FMT_BEGIN_NAMESPACE
+template <typename Char>
+struct formatter<wxString, Char> : basic_ostream_formatter<Char> {};
+FMT_END_NAMESPACE
+
+#endif //KICAD_FMT_H

include/lib_table_base.h

@@ -43,6 +43,7 @@ class LIB_TABLE_LEXER;
 class LIB_ID;
 class LIB_TABLE_ROW;
 class LIB_TABLE_GRID;
+class LIB_TABLE;
 class IO_ERROR;
 
 
@@ -68,7 +69,8 @@ class LIB_TABLE_ROW : boost::noncopyable
 public:
     LIB_TABLE_ROW() :
         enabled( true ),
-        m_loaded( false )
+        m_loaded( false ),
+        m_parent( nullptr )
     {
     }
 
@@ -77,11 +79,12 @@ public:
     }
 
     LIB_TABLE_ROW( const wxString& aNick, const wxString& aURI, const wxString& aOptions,
-                   const wxString& aDescr = wxEmptyString ) :
+                   const wxString& aDescr = wxEmptyString, LIB_TABLE* aParent = nullptr ) :
         nickName( aNick ),
         description( aDescr ),
         enabled( true ),
-        m_loaded( false )
+        m_loaded( false ),
+        m_parent( aParent )
     {
         properties.reset();
         SetOptions( aOptions );
@@ -167,6 +170,10 @@ public:
      */
     void SetDescr( const wxString& aDescr )     { description = aDescr; }
 
+    LIB_TABLE* GetParent() const { return m_parent; }
+
+    void SetParent( LIB_TABLE* aParent ) { m_parent = aParent; }
+
     /**
      * Return the constant #PROPERTIES for this library (#LIB_TABLE_ROW).  These are
      * the "options" in a table.
@@ -198,7 +205,8 @@ protected:
         options( aRow.options ),
         description( aRow.description ),
         enabled( aRow.enabled ),
-        m_loaded( aRow.m_loaded )
+        m_loaded( aRow.m_loaded ),
+        m_parent( aRow.m_parent )
     {
         if( aRow.properties )
             properties = std::make_unique<PROPERTIES>( *aRow.properties.get() );
@@ -225,6 +233,7 @@ private:
 
     bool              enabled  = true;    ///< Whether the LIB_TABLE_ROW is enabled
     bool              m_loaded = false;   ///< Whether the LIB_TABLE_ROW is loaded
+    LIB_TABLE*        m_parent;           ///< Pointer to the table this row lives in (maybe null)
 
     std::unique_ptr< PROPERTIES > properties;
 };
@@ -514,8 +523,6 @@ protected:
 
     void ensureIndex()
     {
-        std::lock_guard<std::recursive_mutex> lock( m_nickIndexMutex );
-
         // The dialog lib table editor may not maintain the nickIndex.
         // Lazy indexing may be required.  To handle lazy indexing, we must enforce
         // that "nickIndex" is either empty or accurate, but never inaccurate.

include/wildcards_and_files_ext.h

@@ -128,6 +128,7 @@ extern const std::string GerberJobFileExtension;
 extern const std::string HtmlFileExtension;
 extern const std::string EquFileExtension;
 extern const std::string HotkeyFileExtension;
+extern const std::string DatabaseLibraryFileExtension;
 
 extern const std::string ArchiveFileExtension;
 
@@ -187,6 +188,7 @@ extern wxString DrawingSheetFileWildcard();
 extern wxString SchematicSymbolFileWildcard();
 extern wxString KiCadSymbolLibFileWildcard();
 extern wxString LegacySymbolLibFileWildcard();
+extern wxString DatabaseLibFileWildcard();
 extern wxString AllSymbolLibFilesWildcard();
 extern wxString ProjectFileWildcard();
 extern wxString LegacyProjectFileWildcard();

qa/data/dblib/qa_dblib.kicad_dbl

@@ -0,0 +1,114 @@
+{
+    "meta": {
+        "version": 0,
+        "filename": "qa_dblib.kicad_dbl"
+    },
+    "name": "QA Database",
+    "description": "A database for testing purposes",
+    "source": {
+        "type": "odbc",
+        "dsn": "",
+        "username": "",
+        "password": "",
+        "timeout_seconds": 2,
+        "connection_string": "Driver={SQLite3};Database=${CWD}/database.sqlite"
+    },
+    "libraries": [
+        {
+            "name": "Resistors",
+            "table": "Resistors",
+            "key": "Part ID",
+            "symbols": "Symbols",
+            "footprints": "Footprints",
+            "fields": [
+                {
+                    "column": "Manufacturer",
+                    "name": "Manufacturer",
+                    "visible_on_add": false,
+                    "visible_in_chooser": false
+                },
+                {
+                    "column": "MPN",
+                    "name": "MPN",
+                    "visible_on_add": false,
+                    "visible_in_chooser": true
+                },
+                {
+                    "column": "Value",
+                    "name": "Value",
+                    "visible_on_add": true,
+                    "visible_in_chooser": true
+                },
+                {
+                    "column": "Resistance",
+                    "name": "Resistance",
+                    "visible_on_add": false,
+                    "visible_in_chooser": false
+                },
+                {
+                    "column": "Resistance Tolerance",
+                    "name": "Tolerance",
+                    "visible_on_add": false,
+                    "visible_in_chooser": false
+                },
+                {
+                    "column": "Power Rating",
+                    "name": "Power Rating",
+                    "visible_on_add": true,
+                    "visible_in_chooser": false
+                },
+                {
+                    "column": "Description",
+                    "name": "ki_description",
+                    "visible_on_add": false,
+                    "visible_in_chooser": true
+                }
+            ]
+        },
+        {
+            "name": "Capacitors",
+            "table": "Capacitors",
+            "key": "Part ID",
+            "symbols": "Symbols",
+            "footprints": "Footprints",
+            "fields": [
+                {
+                    "column": "Manufacturer",
+                    "name": "Manufacturer",
+                    "visible_on_add": false,
+                    "visible_in_chooser": false
+                },
+                {
+                    "column": "MPN",
+                    "name": "MPN",
+                    "visible_on_add": false,
+                    "visible_in_chooser": true
+                },
+                {
+                    "column": "Value",
+                    "name": "Value",
+                    "visible_on_add": true,
+                    "visible_in_chooser": true
+                },
+                {
+                    "column": "Dielectric Type",
+                    "name": "Dielectric",
+                    "visible_on_add": true,
+                    "visible_in_chooser": false
+                },
+                {
+                    "column": "Voltage Rating",
+                    "name": "Voltage Rating",
+                    "visible_on_add": true,
+                    "visible_in_chooser": true
+                },
+                {
+                    "column": "Description",
+                    "name": "ki_description",
+                    "visible_on_add": false,
+                    "visible_in_chooser": true
+                }
+            ]
+        }
+    ]
+}

qa/unittests/common/CMakeLists.txt

@@ -54,6 +54,10 @@ set( QA_COMMON_SRCS
     view/test_zoom_controller.cpp
 )
 
+if( KICAD_TEST_DATABASE_LIBRARIES )
+    set( QA_COMMON_SRCS ${QA_COMMON_SRCS} test_database.cpp )
+endif()
+
 set( QA_COMMON_LIBS
     common
     libcontext
@@ -88,4 +92,10 @@ include_directories(
     ${INC_AFTER}
 )
 
+if( KICAD_TEST_DATABASE_LIBRARIES )
+    set_source_files_properties( test_database.cpp PROPERTIES
+            COMPILE_DEFINITIONS "QA_DATABASE_FILE_LOCATION=(\"${CMAKE_SOURCE_DIR}/qa/data/dblib\")"
+            )
+endif()
+
 kicad_add_boost_test( qa_common qa_common )

qa/unittests/common/test_database.cpp

@@ -0,0 +1,69 @@
+/*
+* This program source code file is part of KiCad, a free EDA CAD application.
+*
+* Copyright (C) 2022 Jon Evans <jon@craftyjon.com>
+* Copyright (C) 2022 KiCad Developers, see AUTHORS.TXT for contributors.
+*
+* This program is free software; you can redistribute it and/or
+* modify it under the terms of the GNU General Public License
+* as published by the Free Software Foundation; either version 2
+* of the License, or (at your option) any later version.
+*
+* This program is distributed in the hope that it will be useful,
+* but WITHOUT ANY WARRANTY; without even the implied warranty of
+* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+* GNU General Public License for more details.
+*
+* You should have received a copy of the GNU General Public License
+* along with this program; if not, you may find one here:
+* http://www.gnu.org/licenses/old-licenses/gpl-2.0.html
+* or you may search the http://www.gnu.org website for the version 2 license,
+* or you may write to the Free Software Foundation, Inc.,
+* 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
+*/
+
+#include <fmt/core.h>
+#include <boost/test/unit_test.hpp>
+
+#include <database/database_connection.h>
+
+BOOST_AUTO_TEST_SUITE( Database )
+
+
+BOOST_AUTO_TEST_CASE( Connect )
+{
+    std::string cs = fmt::format( "Driver={{SQLite3}};Database={}/database.sqlite",
+                                  QA_DATABASE_FILE_LOCATION );
+
+    // Construct and connect
+    DATABASE_CONNECTION dc( cs, 2, false );
+    BOOST_CHECK_NO_THROW( dc.Connect() );
+    BOOST_CHECK( dc.IsConnected() );
+
+    dc.Disconnect();
+    BOOST_CHECK( !dc.IsConnected() );
+
+    dc.Connect();
+    BOOST_CHECK( dc.IsConnected() );
+
+    dc.Disconnect();
+
+    // Scoped connection should self-disconnect
+    {
+        DATABASE_CONNECTION dc2( cs, 2 );
+    }
+
+    dc.Connect();
+    BOOST_CHECK( dc.IsConnected() );
+
+    DATABASE_CONNECTION::ROW result;
+
+    BOOST_CHECK( dc.SelectOne( "Resistors", std::make_pair( "Part ID", "RES-001" ), result ) );
+
+    BOOST_CHECK( !result.empty() );
+    BOOST_CHECK( result.count( "MPN" ) );
+    BOOST_CHECK_NO_THROW( std::any_cast<std::string>( result.at( "MPN" ) ) );
+    BOOST_CHECK_EQUAL( std::any_cast<std::string>( result.at( "MPN" ) ), "RC0603FR-0710KL" );
+}
+
+BOOST_AUTO_TEST_SUITE_END()

thirdparty/nanodbc/CMakeLists.txt

@@ -73,10 +73,10 @@ endif()
 message(STATUS "nanodbc feature: Disable async features - ${NANODBC_DISABLE_ASYNC}")
 
 if(NANODBC_ENABLE_UNICODE)
-  add_definitions(-DNANODBC_ENABLE_UNICODE)
+  add_compile_definitions(NANODBC_ENABLE_UNICODE)
   if(MSVC)
     # Sets "Use Unicode Character Set" property in Visual Studio projects
-    add_definitions(-DUNICODE -D_UNICODE)
+    add_compile_definitions(UNICODE _UNICODE)
   endif()
 endif()
 message(STATUS "nanodbc feature: Enable Unicode - ${NANODBC_ENABLE_UNICODE}")
@@ -179,16 +179,27 @@ endif()
 ########################################
 ## library target
 ########################################
-add_library(nanodbc nanodbc/nanodbc.cpp nanodbc/nanodbc.h)
+add_library(nanodbc
+        STATIC
+        nanodbc/nanodbc.cpp
+        nanodbc/nanodbc.h)
 
 target_link_libraries(nanodbc ${Boost_LIBRARIES} ${ODBC_LIBRARIES})
 
-target_include_directories(nanodbc PUBLIC
+if(APPLE)
+  target_link_libraries(nanodbc ${ODBC_LINK_FLAGS})
+endif()
+
+target_include_directories(nanodbc PUBLIC SYSTEM
   $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
   $<INSTALL_INTERFACE:include/nanodbc>) # <prefix>/include/nanodbc
 
 if(UNIX)
   set_target_properties(nanodbc PROPERTIES
-    COMPILE_FLAGS "${ODBC_CFLAGS}"
+    COMPILE_FLAGS "${ODBC_CFLAGS}")
-    LIBRARY_OUTPUT_DIRECTORY "lib")
 endif()
+
+if(NANODBC_ENABLE_UNICODE)
+  add_compile_definitions(NANODBC_ENABLE_UNICODE)
+  target_compile_definitions(nanodbc PUBLIC NANODBC_ENABLE_UNICODE)
+endif()