kicad-source/gerbview/files.cpp

/*
 * This program source code file is part of KiCad, a free EDA CAD application.
 *
 * Copyright (C) 2017 Jean-Pierre Charras, jp.charras at wanadoo.fr
 * Copyright (C) 2004-2019 KiCad Developers, see AUTHORS.txt for contributors.
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; either version 2
 * of the License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, you may find one here:
 * http://www.gnu.org/licenses/old-licenses/gpl-2.0.html
 * or you may search the http://www.gnu.org website for the version 2 license,
 * or you may write to the Free Software Foundation, Inc.,
 * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
 */

#include <wx/filedlg.h>
#include <wx/wfstream.h>
#include <wx/zipstrm.h>
#include <reporter.h>
#include <dialogs/html_message_box.h>
#include <gerbview_frame.h>
#include <gerbview_id.h>
#include <gerber_file_image.h>
#include <gerber_file_image_list.h>
#include <excellon_image.h>
#include <wildcards_and_files_ext.h>
#include <view/view.h>
#include <widgets/wx_progress_reporters.h>
#include "widgets/gerbview_layer_widget.h"

// HTML Messages used more than one time:
#define MSG_NO_MORE_LAYER _( "<b>No more available layers</b> in GerbView to load files" )
#define MSG_NOT_LOADED _( "<b>Not loaded:</b> <i>%s</i>" )
#define MSG_OOM _( "<b>Memory was exhausted reading:</b> <i>%s</i>" )


void GERBVIEW_FRAME::OnGbrFileHistory( wxCommandEvent& event )
{
    wxString fn;

    fn = GetFileFromHistory( event.GetId(), _( "Gerber files" ) );

    if( !fn.IsEmpty() )
    {
        LoadGerberFiles( fn );
    }
}

void GERBVIEW_FRAME::OnClearGbrFileHistory( wxCommandEvent& aEvent )
{
    ClearFileHistory();
}


void GERBVIEW_FRAME::OnDrlFileHistory( wxCommandEvent& event )
{
    wxString fn;

    fn = GetFileFromHistory( event.GetId(), _( "Drill files" ), &m_drillFileHistory );

    if( !fn.IsEmpty() )
    {
        LoadExcellonFiles( fn );
    }
}


void GERBVIEW_FRAME::OnClearDrlFileHistory( wxCommandEvent& aEvent )
{
    ClearFileHistory( &m_drillFileHistory );
}


void GERBVIEW_FRAME::OnZipFileHistory( wxCommandEvent& event )
{
    wxString filename;
    filename = GetFileFromHistory( event.GetId(), _( "Zip files" ), &m_zipFileHistory );

    if( !filename.IsEmpty() )
    {
        LoadZipArchiveFile( filename );
    }
}


void GERBVIEW_FRAME::OnClearZipFileHistory( wxCommandEvent& aEvent )
{
    ClearFileHistory( &m_zipFileHistory );
}


void GERBVIEW_FRAME::OnJobFileHistory( wxCommandEvent& event )
{
    wxString filename = GetFileFromHistory( event.GetId(), _( "Job files" ), &m_jobFileHistory );

    if( !filename.IsEmpty() )
        LoadGerberJobFile( filename );
}


void GERBVIEW_FRAME::OnClearJobFileHistory( wxCommandEvent& aEvent )
{
    ClearFileHistory( &m_jobFileHistory );
}


bool GERBVIEW_FRAME::LoadGerberFiles( const wxString& aFullFileName )
{
    static int lastGerberFileWildcard = 0;
    wxString   filetypes;
    wxArrayString filenamesList;
    wxFileName filename = aFullFileName;
    wxString currentPath;

    if( !filename.IsOk() )
    {
        /* Standard gerber filetypes
         * (See http://en.wikipedia.org/wiki/Gerber_File)
         * The .gbr (.pho in legacy files) extension is the default used in Pcbnew; however
         * there are a lot of other extensions used for gerber files.  Because the first letter
         * is usually g, we accept g* as extension.
         * (Mainly internal copper layers do not have specific extension, and filenames are like
         * *.g1, *.g2 *.gb1 ...)
         * Now (2014) Ucamco (the company which manages the Gerber format) encourages use of .gbr
         * only and the Gerber X2 file format.
         */
        filetypes = _( "Gerber files (.g* .lgr .pho)" );
        filetypes << wxT("|");
        filetypes += wxT("*.g*;*.G*;*.pho;*.PHO" );
        filetypes << wxT("|");

        /* Special gerber filetypes */
        filetypes += _( "Top layer" ) + AddFileExtListToFilter( { "GTL" } ) + wxT( "|" );
        filetypes += _( "Bottom layer" ) + AddFileExtListToFilter( { "GBL" } ) + wxT( "|" );
        filetypes += _( "Bottom solder resist" ) + AddFileExtListToFilter( { "GBS" } ) + wxT( "|" );
        filetypes += _( "Top solder resist" ) + AddFileExtListToFilter( { "GTS" } ) + wxT( "|" );
        filetypes += _( "Bottom overlay" ) + AddFileExtListToFilter( { "GBO" } ) + wxT( "|" );
        filetypes += _( "Top overlay" ) + AddFileExtListToFilter( { "GTO" } ) + wxT( "|" );
        filetypes += _( "Bottom paste" ) + AddFileExtListToFilter( { "GBP" } ) + wxT( "|" );
        filetypes += _( "Top paste" ) + AddFileExtListToFilter( { "GTP" } ) + wxT( "|" );
        filetypes += _( "Keep-out layer" ) + AddFileExtListToFilter( { "GKO" } ) + wxT( "|" );
        filetypes += _( "Mechanical layers" ) + AddFileExtListToFilter( { "GM1", "GM2", "GM3", "GM4", "GM5", "GM6", "GM7", "GM8", "GM9" } ) + wxT( "|" );
        filetypes += _( "Top Pad Master" ) + AddFileExtListToFilter( { "GPT" } ) + wxT( "|" );
        filetypes += _( "Bottom Pad Master" ) + AddFileExtListToFilter( { "GPB" } ) + wxT( "|" );

        // All filetypes
        filetypes += AllFilesWildcard();

        // Use the current working directory if the file name path does not exist.
        if( filename.DirExists() )
            currentPath = filename.GetPath();
        else
        {
            currentPath = m_mruPath;

            // On wxWidgets 3.1 (bug?) the path in wxFileDialog is ignored when
            // finishing by the dir separator. Remove it if any:
            if( currentPath.EndsWith( '\\' ) || currentPath.EndsWith( '/' ) )
                currentPath.RemoveLast();
        }

        wxFileDialog dlg( this, _( "Open Gerber File(s)" ), currentPath, filename.GetFullName(),
                          filetypes,
                          wxFD_OPEN | wxFD_FILE_MUST_EXIST | wxFD_MULTIPLE | wxFD_CHANGE_DIR );
        dlg.SetFilterIndex( lastGerberFileWildcard );

        if( dlg.ShowModal() == wxID_CANCEL )
            return false;

        lastGerberFileWildcard = dlg.GetFilterIndex();
        dlg.GetPaths( filenamesList );
        m_mruPath = currentPath = dlg.GetDirectory();
    }
    else
    {
        filenamesList.Add( aFullFileName );
        m_mruPath = currentPath = filename.GetPath();
    }


    // Set the busy cursor
    wxBusyCursor wait;

    bool isFirstFile = GetImagesList()->GetLoadedImageCount() == 0;

    bool success = LoadListOfGerberAndDrillFiles( currentPath, filenamesList );

    // Auto zoom is only applied if there is only one file loaded
    if( isFirstFile )
    {
        Zoom_Automatique( false );
    }

    return success;
}


bool GERBVIEW_FRAME::LoadListOfGerberAndDrillFiles( const wxString& aPath,
                                            const wxArrayString& aFilenameList,
                                            const std::vector<int>* aFileType )
{
    wxFileName filename;

    // Read gerber files: each file is loaded on a new GerbView layer
    bool success = true;
    int layer = GetActiveLayer();
    int  firstLoadedLayer = NO_AVAILABLE_LAYERS;
    LSET visibility = GetVisibleLayers();

    // Manage errors when loading files
    wxString msg;
    WX_STRING_REPORTER reporter( &msg );

    // Create progress dialog (only used if more than 1 file to load
    std::unique_ptr<WX_PROGRESS_REPORTER> progress = nullptr;

    for( unsigned ii = 0; ii < aFilenameList.GetCount(); ii++ )
    {
        filename = aFilenameList[ii];

        if( !filename.IsAbsolute() )
            filename.SetPath( aPath );

        // Check for non existing files, to avoid creating broken or useless data
        // and report all in one error list:
        if( !filename.FileExists() )
        {
            wxString warning;
            warning << "<b>" << _( "File not found:" ) << "</b><br>"
                    << filename.GetFullPath() << "<br>";
            reporter.Report( warning, RPT_SEVERITY_WARNING );
            success = false;
            continue;
        }

        if( filename.GetExt() == GerberJobFileExtension.c_str() )
        {
            //We cannot read a gerber job file as a gerber plot file: skip it
            wxString txt;
            txt.Printf( _( "<b>A gerber job file cannot be loaded as a plot file</b> "
                           "<i>%s</i>" ),
                        filename.GetFullName() );
            success = false;
            reporter.Report( txt, RPT_SEVERITY_ERROR );
            continue;
        }


        m_lastFileName = filename.GetFullPath();

        if( !progress && ( aFilenameList.GetCount() > 1 ) )
        {
            progress = std::make_unique<WX_PROGRESS_REPORTER>( this,
                                                               _( "Loading Gerber files..." ), 1,
                                                               false );
            progress->SetMaxProgress( aFilenameList.GetCount() - 1 );
            progress->Report( wxString::Format( _("Loading %u/%zu %s..." ),
                                                ii+1,
                                                aFilenameList.GetCount(),
                                                m_lastFileName ) );
        }
        else if( progress )
        {
            progress->Report( wxString::Format( _("Loading %u/%zu %s..." ),
                                                ii+1,
                                                aFilenameList.GetCount(),
                                                m_lastFileName ) );
            progress->KeepRefreshing();
        }


        // Make sure we have a layer available to load into
        layer = getNextAvailableLayer();

        if( layer == NO_AVAILABLE_LAYERS )
        {
            success = false;
            reporter.Report( MSG_NO_MORE_LAYER, RPT_SEVERITY_ERROR );

            // Report the name of not loaded files:
            while( ii < aFilenameList.GetCount() )
            {
                filename = aFilenameList[ii++];
                wxString txt = wxString::Format( MSG_NOT_LOADED, filename.GetFullName() );
                reporter.Report( txt, RPT_SEVERITY_ERROR );
            }
            break;
        }

        SetActiveLayer( layer, false );
        visibility[ layer ] = true;

        try
        {
            if( aFileType && ( *aFileType )[ii] == 1 )
            {
                if( LoadExcellonFiles( filename.GetFullPath() ) )
                {
                    UpdateFileHistory( filename.GetFullPath(), &m_drillFileHistory );

                    // Select the first added layer by default when done loading
                    if( firstLoadedLayer == NO_AVAILABLE_LAYERS )
                    {
                        firstLoadedLayer = layer;
                    }
                }
            }
            else
            {
                if( Read_GERBER_File( filename.GetFullPath() ) )
                {
                    UpdateFileHistory( filename.GetFullPath() );

                    if( firstLoadedLayer == NO_AVAILABLE_LAYERS )
                    {
                        firstLoadedLayer = layer;
                    }
                }
            }
        }
        catch( const std::bad_alloc& )
        {
            wxString txt = wxString::Format( MSG_OOM, filename.GetFullName() );
            reporter.Report( txt, RPT_SEVERITY_ERROR );
            success = false;
            continue;
        }

        if( progress )
            progress->AdvanceProgress();
    }

    if( !success )
    {
        wxSafeYield();  // Allows slice of time to redraw the screen
                        // to refresh widgets, before displaying messages
        HTML_MESSAGE_BOX mbox( this, _( "Errors" ) );
        mbox.ListSet( msg );
        mbox.ShowModal();
    }

    SetVisibleLayers( visibility );

    if( firstLoadedLayer != NO_AVAILABLE_LAYERS )
        SetActiveLayer( firstLoadedLayer, true );

    // Synchronize layers tools with actual active layer:
    ReFillLayerWidget();

    m_LayersManager->UpdateLayerIcons();
    syncLayerBox( true );

    GetCanvas()->Refresh();

    return success;
}


bool GERBVIEW_FRAME::LoadExcellonFiles( const wxString& aFullFileName )
{
    wxString   filetypes;
    wxArrayString filenamesList;
    wxFileName filename = aFullFileName;
    wxString currentPath;

    if( !filename.IsOk() )
    {
        filetypes = DrillFileWildcard();
        filetypes << wxT( "|" );

        /* All filetypes */
        filetypes += AllFilesWildcard();

        /* Use the current working directory if the file name path does not exist. */
        if( filename.DirExists() )
            currentPath = filename.GetPath();
        else
            currentPath = m_mruPath;

        wxFileDialog dlg( this, _( "Open NC (Excellon) Drill File(s)" ),
                          currentPath, filename.GetFullName(), filetypes,
                          wxFD_OPEN | wxFD_FILE_MUST_EXIST | wxFD_MULTIPLE | wxFD_CHANGE_DIR );

        if( dlg.ShowModal() == wxID_CANCEL )
            return false;

        dlg.GetPaths( filenamesList );
        currentPath = wxGetCwd();
        m_mruPath = currentPath;
    }
    else
    {
        filenamesList.Add( aFullFileName );
        currentPath = filename.GetPath();
        m_mruPath = currentPath;
    }

    // Read Excellon drill files: each file is loaded on a new GerbView layer
    bool success = true;
    int layer = GetActiveLayer();

    // Manage errors when loading files
    wxString msg;
    WX_STRING_REPORTER reporter( &msg );

    for( unsigned ii = 0; ii < filenamesList.GetCount(); ii++ )
    {
        filename = filenamesList[ii];

        if( !filename.IsAbsolute() )
            filename.SetPath( currentPath );

        m_lastFileName = filename.GetFullPath();

        layer = getNextAvailableLayer();

        if( layer == NO_AVAILABLE_LAYERS )
        {
            success = false;
            reporter.Report( MSG_NO_MORE_LAYER, RPT_SEVERITY_ERROR );

            // Report the name of not loaded files:
            while( ii < filenamesList.GetCount() )
            {
                filename = filenamesList[ii++];
                wxString txt = wxString::Format( MSG_NOT_LOADED, filename.GetFullName() );
                reporter.Report( txt, RPT_SEVERITY_ERROR );
            }
            break;
        }
        else
        {
            SetActiveLayer( layer, false );

            if( Read_EXCELLON_File( filename.GetFullPath() ) )
            {
                // Update the list of recent drill files.
                UpdateFileHistory( filename.GetFullPath(), &m_drillFileHistory );
            }
            else
            {
                wxString txt = wxString::Format( MSG_NOT_LOADED, filename.GetFullName() );
                reporter.Report( txt, RPT_SEVERITY_ERROR );
            }
        }
    }

    if( !success )
    {
        HTML_MESSAGE_BOX mbox( this, _( "Errors" ) );
        mbox.ListSet( msg );
        mbox.ShowModal();
    }

    Zoom_Automatique( false );

    // Synchronize layers tools with actual active layer:
    ReFillLayerWidget();
    SetActiveLayer( GetActiveLayer() );
    m_LayersManager->UpdateLayerIcons();
    syncLayerBox();

    return success;
}


bool GERBVIEW_FRAME::unarchiveFiles( const wxString& aFullFileName, REPORTER* aReporter )
{
    bool     foundX2Gerbers = false;
    wxString msg;
    int      firstLoadedLayer = NO_AVAILABLE_LAYERS;

    // Extract the path of aFullFileName. We use it to store temporary files
    wxFileName fn( aFullFileName );
    wxString unzipDir = fn.GetPath();

    wxFFileInputStream zipFile( aFullFileName );

    if( !zipFile.IsOk() )
    {
        if( aReporter )
        {
            msg.Printf( _( "Zip file '%s' cannot be opened." ), aFullFileName );
            aReporter->Report( msg, RPT_SEVERITY_ERROR );
        }

        return false;
    }

    // Update the list of recent zip files.
    UpdateFileHistory( aFullFileName, &m_zipFileHistory );

    // The unzipped file in only a temporary file. Give it a filename
    // which cannot conflict with an usual filename.
    // TODO: make Read_GERBER_File() and Read_EXCELLON_File() able to
    // accept a stream, and avoid using a temp file.
    wxFileName temp_fn( "$tempfile.tmp" );
    temp_fn.MakeAbsolute( unzipDir );
    wxString unzipped_tempfile = temp_fn.GetFullPath();


    bool success = true;
    wxZipInputStream zipArchive( zipFile );
    wxZipEntry* entry;
    bool reported_no_more_layer = false;

    while( ( entry = zipArchive.GetNextEntry() ) )
    {
        wxString fname = entry->GetName();
        wxFileName uzfn = fname;
        wxString curr_ext = uzfn.GetExt().Lower();

        // The archive contains Gerber and/or Excellon drill files. Use the right loader.
        // However it can contain a few other files (reports, pdf files...),
        // which will be skipped.
        if( curr_ext == GerberJobFileExtension.c_str() )
        {
            //We cannot read a gerber job file as a gerber plot file: skip it
            if( aReporter )
            {
                msg.Printf( _( "Skipped file '%s' (gerber job file)." ), entry->GetName() );
                aReporter->Report( msg, RPT_SEVERITY_WARNING );
            }

            continue;
        }

        wxString               matchedExt;
        enum GERBER_ORDER_ENUM order;
        GERBER_FILE_IMAGE_LIST::GetGerberLayerFromFilename( fname, order, matchedExt );

        int layer = GetActiveLayer();

        if( layer == NO_AVAILABLE_LAYERS )
        {
            success = false;

            if( aReporter )
            {
                if( !reported_no_more_layer )
                    aReporter->Report( MSG_NO_MORE_LAYER,  RPT_SEVERITY_ERROR );

                reported_no_more_layer = true;

                // Report the name of not loaded files:
                msg.Printf( MSG_NOT_LOADED, entry->GetName() );
                aReporter->Report( msg, RPT_SEVERITY_ERROR );
            }

            delete entry;
            continue;
        }

        // Create the unzipped temporary file:
        {
            wxFFileOutputStream temporary_ofile( unzipped_tempfile );

            if( temporary_ofile.Ok() )
                temporary_ofile.Write( zipArchive );
            else
            {
                success = false;

                if( aReporter )
                {
                    msg.Printf( _( "<b>Unable to create temporary file '%s'.</b>" ),
                                unzipped_tempfile );
                    aReporter->Report( msg, RPT_SEVERITY_ERROR );
                }
            }
        }

        bool read_ok = true;

        // Try to parse files if we can't tell from file extension
        if( order == GERBER_ORDER_ENUM::GERBER_LAYER_UNKNOWN )
        {
            if( EXCELLON_IMAGE::TestFileIsExcellon( unzipped_tempfile ) )
            {
                order = GERBER_ORDER_ENUM::GERBER_DRILL;
            }
            else if( GERBER_FILE_IMAGE::TestFileIsRS274( unzipped_tempfile ) )
            {
                // If we have no way to know what layer it is, just guess
                order = GERBER_ORDER_ENUM::GERBER_TOP_COPPER;
            }
            else
            {
                if( aReporter )
                {
                    msg.Printf( _( "Skipped file '%s' (unknown type)." ), entry->GetName() );
                    aReporter->Report( msg, RPT_SEVERITY_WARNING );
                }
            }
        }

        if( order == GERBER_ORDER_ENUM::GERBER_DRILL )
        {
            read_ok = Read_EXCELLON_File( unzipped_tempfile );
        }
        else if( order != GERBER_ORDER_ENUM::GERBER_LAYER_UNKNOWN )
        {
            // Read gerber files: each file is loaded on a new GerbView layer
            read_ok = Read_GERBER_File( unzipped_tempfile );

            if( read_ok )
                GetCanvas()->GetView()->SetLayerHasNegatives(
                        GERBER_DRAW_LAYER( layer ), GetGbrImage( layer )->HasNegativeItems() );
        }

        // Select the first added layer by default when done loading
        if( read_ok && firstLoadedLayer == NO_AVAILABLE_LAYERS )
        {
            firstLoadedLayer = layer;
        }

        delete entry;

        // The unzipped file is only a temporary file, delete it.
        wxRemoveFile( unzipped_tempfile );

        if( !read_ok )
        {
            success = false;

            if( aReporter )
            {
                msg.Printf( _( "<b>unzipped file %s read error</b>" ), unzipped_tempfile );
                aReporter->Report( msg, RPT_SEVERITY_ERROR );
            }
        }
        else
        {
            GERBER_FILE_IMAGE* gerber_image = GetGbrImage( layer );

            if( gerber_image )
            {
                gerber_image->m_FileName = fname;
                if( gerber_image->m_IsX2_file )
                    foundX2Gerbers = true;
            }

            layer = getNextAvailableLayer();
            SetActiveLayer( layer, false );
        }
    }

    if( foundX2Gerbers )
        SortLayersByX2Attributes();
    else
        SortLayersByFileExtension();

    // Select the first layer loaded so we don't show another layer on top after
    if( firstLoadedLayer != NO_AVAILABLE_LAYERS )
        SetActiveLayer( firstLoadedLayer, true );

    return success;
}


bool GERBVIEW_FRAME::LoadZipArchiveFile( const wxString& aFullFileName )
{
#define ZipFileExtension "zip"

    wxFileName filename = aFullFileName;
    wxString currentPath;

    if( !filename.IsOk() )
    {
        // Use the current working directory if the file name path does not exist.
        if( filename.DirExists() )
            currentPath = filename.GetPath();
        else
            currentPath = m_mruPath;

        wxFileDialog dlg( this, _( "Open Zip File" ), currentPath, filename.GetFullName(),
                          ZipFileWildcard(), wxFD_OPEN | wxFD_FILE_MUST_EXIST | wxFD_CHANGE_DIR );

        if( dlg.ShowModal() == wxID_CANCEL )
            return false;

        filename = dlg.GetPath();
        currentPath = wxGetCwd();
        m_mruPath = currentPath;
    }
    else
    {
        currentPath = filename.GetPath();
        m_mruPath = currentPath;
    }

    wxString msg;
    WX_STRING_REPORTER reporter( &msg );

    if( filename.IsOk() )
        unarchiveFiles( filename.GetFullPath(), &reporter );

    Zoom_Automatique( false );

    // Synchronize layers tools with actual active layer:
    ReFillLayerWidget();
    SetActiveLayer( GetActiveLayer() );
    m_LayersManager->UpdateLayerIcons();
    syncLayerBox();

    if( !msg.IsEmpty() )
    {
        wxSafeYield();  // Allows slice of time to redraw the screen
                        // to refresh widgets, before displaying messages
        HTML_MESSAGE_BOX mbox( this, _( "Messages" ) );
        mbox.ListSet( msg );
        mbox.ShowModal();
    }

    return true;
}