pythonEngine.h

/* ********************************************************************
    itom software
    URL: http://www.uni-stuttgart.de/ito
    Copyright (C) 2024, Institut für Technische Optik (ITO),
    Universität Stuttgart, Germany
 
    This file is part of itom.
 
    itom is free software; you can redistribute it and/or modify it
    under the terms of the GNU Library General Public Licence as published by
    the Free Software Foundation; either version 2 of the Licence, or (at
    your option) any later version.
 
    itom is distributed in the hope that it will be useful, but
    WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Library
    General Public Licence for more details.
 
    You should have received a copy of the GNU Library General Public License
    along with itom. If not, see <http://www.gnu.org/licenses/>.
*********************************************************************** */
 
#pragma once
 
/*if you add any include to this file you will DIE an immediate, horrible, painful death*/
 
#include <string>
#include <exception>
//#ifndef Q_MOC_RUN
//    #define PY_ARRAY_UNIQUE_SYMBOL itom_ARRAY_API
//    #define NO_IMPORT_ARRAY
//#endif
 
#include "pythonJedi.h"
 
#ifndef Q_MOC_RUN
    #include "pythonWrapper.h"
 
    //python
    // see http://vtk.org/gitweb?p=VTK.git;a=commitdiff;h=7f3f750596a105d48ea84ebfe1b1c4ca03e0bab3
    #if (defined _DEBUG) && (defined WIN32)
        // see https://github.com/microsoft/onnxruntime/issues/9735#issuecomment-970718821
        #include <corecrt.h>
        #undef _DEBUG
        #if (PY_VERSION_HEX < 0x03070000)
            #include "node.h"
        #endif
        #define _DEBUG
    #else
        #if (PY_VERSION_HEX < 0x03070000)
            #include "node.h"
        #endif
    #endif
 
#endif // Q_MOC_RUN
 
/* includes */
 
#include "pythonItom.h"
#include "pythonProxy.h"
#include "pythonStream.h"
#include "pythonCommon.h"
#include "pythonJediRunner.h"
 
#include "../models/breakPointModel.h"
#include "../../common/sharedStructuresQt.h"
#include "../../common/addInInterface.h"
#include "../../common/functionCancellationAndObserver.h"
#include "../codeEditor/codeCheckerItem.h"
#include "../global.h"
 
#include "pythonWorkspace.h"
 
#include <qstringlist.h>
#include <qqueue.h>
#include <qset.h>
#include <qpointer.h>
#include <qatomic.h>
#include <qelapsedtimer.h>
#include <qtimer.h>
#include <qsettings.h>
 
/* definition and macros */
 
/* global variables (avoid) */
 
/* content */
 
class QDesktopWidget;
class QTimer;
 
namespace ito
{
 
class FuncWeakRef
{
public:
    FuncWeakRef();
    FuncWeakRef(PythonProxy::PyProxy *proxyObject, PyObject *argTuple = NULL);
    FuncWeakRef(const FuncWeakRef &rhs);
    ~FuncWeakRef();
    FuncWeakRef& operator =(FuncWeakRef rhs);
 
    PythonProxy::PyProxy* getProxyObject() const { return m_proxyObject; } //borrowed reference
    PyObject* getArguments() const { return m_argument; } //borrowed reference
    bool isValid() const { return (m_proxyObject != NULL); }
 
    void setHandle(const size_t &handle);
    size_t getHandle() const { return m_handle; }
private:
    PythonProxy::PyProxy *m_proxyObject;
    PyObject *m_argument;
    size_t m_handle;
};
 
 
class PythonEngine : public QObject
{
    Q_OBJECT
 
public:
    PythonEngine();                                 //constructor
    ~PythonEngine();                                //destructor
 
    Q_INVOKABLE void pythonSetup(ito::RetVal *retValue, QSharedPointer<QVariantMap> infoMessages);               //setup
 
    Q_INVOKABLE ito::RetVal scanAndRunAutostartFolder(QString currentDirAfterScan = QString() );
    Q_INVOKABLE ito::RetVal pythonShutdown(ItomSharedSemaphore *aimWait = NULL);            //shutdown
    Q_INVOKABLE ito::RetVal stringEncodingChanged();
 
    inline ito::BreakPointModel *getBreakPointModel() const { return m_bpModel; }
    inline bool isPythonBusy() const                { return m_pythonState != ito::pyStateIdle; }
    inline bool isPythonDebugging() const           { return (m_pythonState == ito::pyStateDebuggingWaitingButBusy || m_pythonState == ito::pyStateDebugging || m_pythonState == ito::pyStateDebuggingWaiting); }
    inline bool isPythonDebuggingAndWaiting() const { return m_pythonState == ito::pyStateDebuggingWaiting; }
    inline bool execInternalCodeByDebugger() const  { return m_executeInternalPythonCodeInDebugMode; }
    inline void setExecInternalCodeByDebugger(bool value) { m_executeInternalPythonCodeInDebugMode = value; }
    void printPythonErrorWithoutTraceback();
    void pythonDebugFunction(PyObject *callable, PyObject *argTuple, bool gilExternal = false);
    void pythonRunFunction(PyObject *callable, PyObject *argTuple, bool gilExternal = false);
    inline PyObject *getGlobalDictionary()  const { return m_pGlobalDictionary;  }  
    inline bool pySyntaxCheckAvailable() const { return (m_pyModCodeChecker != NULL); }
    bool tryToLoadJediIfNotYetDone(); //returns true, if Jedi is already loaded or could be loaded; else false
 
    /* From Python 3.7 on, the AST is parsed to check for the major blocks. Then the encoding is always empty.
    Before, PyParser_SimpleParseString is used with the old Python compiler and an optional encoidng can be detected.
 
    This methods acquires the GIL, therefore it must directly called from a non-python thread.
    */
    QList<int> parseAndSplitCommandInMainComponents(const QString &str, QByteArray &encoding) const;
 
    QString getPythonExecutable() const { return m_pythonExecutable; }
    Qt::HANDLE getPythonThreadId() const { return m_pythonThreadId; }
 
    void enqueueJediCompletionRequest(const ito::JediCompletionRequest &request);
 
    void enqueueJediCalltipRequest(const ito::JediCalltipRequest &request);
 
    void enqueueGoToAssignmentRequest(const ito::JediAssignmentRequest &request);
 
    void enqueueJediGetHelpRequest(const ito::JediGetHelpRequest &request);
 
    void enqueueJediRenameRequest(const ito::JediRenameRequest& request);
 
    static bool isInterruptQueued();
    static const PythonEngine *getInstance();
 
    void addFunctionCancellationAndObserver(QWeakPointer<ito::FunctionCancellationAndObserver> observer);
 
    void removeFunctionCancellationAndObserver(ito::FunctionCancellationAndObserver* observer = NULL);
 
protected:
    ito::RetVal runPyFile(const QString &pythonFileName);         // run file pythonFileName
    ito::RetVal debugFile(const QString &pythonFileName);         // debug file pythonFileName
    ito::RetVal runString(const QString &command);          // run string command
    ito::RetVal debugString(const QString &command);        // debug string command
    ito::RetVal debugFunction(PyObject *callable, PyObject *argTuple, bool gilExternal = false);
    ito::RetVal runFunction(PyObject *callable, PyObject *argTuple, bool gilExternal = false);
 
    ito::RetVal modifyTracebackDepth(int NrOfLevelsToPopAtFront = -1, bool showTraceback = true);
 
    PyObject* setPyErrFromException(const std::exception &exc);
    void startupInitPythonWorkspaceUpdateQueue();
    ito::RetVal startupInitPythonHelpStreamConsumer(QSettings& settings);
    ito::RetVal startupAddModulesToItomModule();
    ito::RetVal startupLoadAndImportAdditionalModules(QSharedPointer<QVariantMap>& infoMessages);
    void connectNotify(const QMetaMethod &signal);
 
    enum DebuggerErrorCode
    {
        DbgErrorNo = 0,
        DbgErrorInvalidBp = 1, // the breakpoint candidate could not be set and will be deleted
        DbgErrorOther = 2 // any other error
    };
 
private:
    enum DictUpdateFlag
    {
        DictUpdate,
        DictReset,
        DictNoAction
    };
 
    struct PythonWorkspaceUpdateQueue
    {
        PythonWorkspaceUpdateQueue() :
            actionGlobal(DictNoAction),
            actionLocal(DictNoAction),
            timerElapsedSinceFirstAction(nullptr),
            timerElapsedSinceLastUpdate(nullptr)
        {
        }
 
        void reset()
        {
            actionGlobal = DictNoAction;
            actionLocal = DictNoAction;
            timerElapsedSinceLastUpdate->stop();
            timerElapsedSinceFirstAction->stop();
        }
 
        QTimer *timerElapsedSinceFirstAction;
 
        QTimer *timerElapsedSinceLastUpdate;
 
        DictUpdateFlag actionGlobal;
        DictUpdateFlag actionLocal;
    };
 
    static PythonEngine *getInstanceInternal();
 
    inline PyObject *getLocalDictionary() { return m_pLocalDictionary; }
 
    PyObject *getPyObjectByFullName(bool globalNotLocal, const QStringList &fullNameSplittedByDelimiter, QString *validVariableName = NULL); //Python GIL must be locked when calling this function!
    PyObject *getPyObjectByFullName(bool globalNotLocal, const QString &fullName, QString *validVariableName = NULL); //Python GIL must be locked when calling this function!
 
    void setGlobalDictionary(PyObject* mainDict = NULL);
    void setLocalDictionary(PyObject* localDict);
 
    void updatePythonWorkspaces(DictUpdateFlag globalDict, DictUpdateFlag localDict, bool lockGIL, bool delayExecution = false);
 
    ito::RetVal pickleDictionary(PyObject *dict, const QString &filename);
    ito::RetVal unpickleDictionary(PyObject *destinationDict, const QString &filename, bool overwrite, bool replaceKeyByValidPyIdentifier);
 
    void pythonRunString(QString cmd);
 
    void pythonDebugString(QString cmd);
 
    QString modifyCommandStringInCaseOfSpecialComments(const QString& command);
 
    //methods for debugging
    void enqueueDbgCmd(ito::tPythonDbgCmd dbgCmd);
    ito::tPythonDbgCmd dequeueDbgCmd();
    bool DbgCommandsAvailable();
    void clearDbgCmdLoop();
 
    ito::RetVal pythonStateTransition(tPythonTransitions transition, bool immediate = true);
 
    //methods for breakpoint
    ito::RetVal pythonAddBreakpoint(const BreakPointItem &breakpoint, int &pyBpNumber);
    ito::RetVal pythonEditBreakpoint(const int pyBpNumber, const BreakPointItem &newBreakpoint);
    ito::RetVal pythonDeleteBreakpoint(const int pyBpNumber);
    void submitAllBreakpointsToDebugger();
 
    ito::RetVal autoReloaderCheck();
 
    static int queuedInterrupt(void *arg);
 
    PyObject* getAndCheckIdentifier(const QString &identifier, ito::RetVal &retval) const;
 
        QVariantMap checkCodeCheckerRequirements();
    ito::RetVal handlePythonSysExit();
 
        struct CodeCheckerOptions
        {
        PythonCommon::CodeCheckerMode mode;
                bool includeItomModuleBeforeCheck;
        PythonCommon::CodeCheckerMessageType minVisibleMessageTypeLevel; 
        QByteArray furtherPropertiesJson; 
        };
 
    //member variables
    bool m_started;
        CodeCheckerOptions m_codeCheckerOptions;
 
    QMutex m_dbgCmdMutex;
    QMutex m_pythonStateChangeMutex;
    QQueue<ito::tPythonDbgCmd> m_debugCommandQueue;
 
    ito::tPythonState m_pythonState;
 
    ito::BreakPointModel *m_bpModel;
 
    PyObject* m_mainModule;          
    PyObject* m_pMainDictionary;   
    PyObject* m_pLocalDictionary;  
    PyObject* m_pGlobalDictionary; 
    PyObject *m_itomDbgModule;       
    PyObject *m_itomDbgInstance;     
    PyObject *m_itomModule;          
    PyObject *m_itomFunctions;       
    PyObject *m_pyModGC;
    PyObject *m_pyModCodeChecker;
        bool m_pyModCodeCheckerHasPyFlakes; 
        bool m_pyModCodeCheckerHasFlake8; 
 
    QSharedPointer<PythonJediRunner> m_jediRunner;
    Qt::HANDLE m_pythonThreadId;
    PyObject *m_dictUnicode;
    PyObject *m_slotsUnicode;
 
 
    QSet<ito::PyWorkspaceContainer*> m_mainWorkspaceContainer;
    QSet<ito::PyWorkspaceContainer*> m_localWorkspaceContainer;
    QHash<size_t, FuncWeakRef> m_pyFuncWeakRefHashes; 
    size_t m_pyFuncWeakRefAutoInc;
 
    QString m_pythonExecutable; 
 
    bool m_executeInternalPythonCodeInDebugMode; 
 
 
    bool m_includeItomImportBeforeCodeAnalysis;
 
    QString m_includeItomImportString;
 
    wchar_t *m_pUserDefinedPythonHome;
 
    QList<QWeakPointer<ito::FunctionCancellationAndObserver> > m_activeFunctionCancellations;
 
    struct AutoReload
    {
        PyObject *modAutoReload;
        PyObject *classAutoReload;
        bool enabled;
        bool checkFileExec;
        bool checkStringExec;
        bool checkFctExec;
    };
 
    AutoReload m_autoReload;
 
    PythonWorkspaceUpdateQueue m_pyWorkspaceUpdateQueue;
 
    static PyMethodDef PyMethodItomDbg[];
    static PyModuleDef PyModuleItomDbg;
    static PyObject* PyInitItomDbg(void);
    static PyObject* PyDbgCommandLoop(PyObject *pSelf, PyObject *pArgs);
    static PyObject* PyDbgClearBreakpoint(PyObject* pSelf, PyObject* pArgs);
 
    //other static members
    static QMutex instancePtrProtection;
    static QString fctHashPrefix;
    static PythonEngine* instance;
 
    QAtomicInt m_interruptCounter; //protects that a python interrupt can only be placed if there is no interrupt event queued yet.
 
    // friend class
    friend class ito::PythonItom;
        friend class ito::PyStream;
 
signals:
    void pythonDebugPositionChanged(QString filename, int lineNo);
    void pythonStateChanged(tPythonTransitions pyTransition, bool immediate);
    void pythonModifyLocalDict(PyObject* localDict, ItomSharedSemaphore* semaphore);
    void pythonModifyGlobalDict(PyObject* globalDict, ItomSharedSemaphore* semaphore);
    void pythonCurrentDirChanged();
    void updateCallStack(QStringList filenames, IntList lines, QStringList methods);
    void deleteCallStack();
 
    void pythonSetCursor(const Qt::CursorShape cursor);
    void pythonResetCursor();
    void pythonAutoReloadChanged(bool enabled, bool checkFile, bool checkCmd, bool checkFct);
    void clearCommandLine();
    void interruptCommandInput();
    void startInputCommandLine(QSharedPointer<QByteArray> buffer, ItomSharedSemaphore *semaphore);
 
private slots:
    void processPythonWorkspaceUpdateQueue();
 
public slots:
    void pythonExecStringFromCommandLine(QString cmd);
    void pythonRunFile(QString filename);
    void pythonDebugFile(QString filename);
    void pythonRunStringOrFunction(QString cmdOrFctHash);
    void pythonDebugStringOrFunction(QString cmdOrFctHash);
    void pythonInterruptExecutionThreadSafe(bool *interruptActuatorsAndTimers = NULL); //if interruptActuatorsAndTimers is NULL, the itom settings will be read, else this boolean variable decides
    void pythonDebugCommand(tPythonDbgCmd cmd);
 
    void setAutoReloader(bool enabled, bool checkFile, bool checkCmd, bool checkFct);
 
    // Settings are necessary for automatic itom inclusion and syntax check
    void readSettings();
    void propertiesChanged();
 
    void pythonCodeCheck(const QString &code, const QString &filename, bool fileSaved, QPointer<QObject> sender, QByteArray callbackFctName);
 
    void breakPointAdded(BreakPointItem bp, int row);
    void breakPointDeleted(QString filename, int lineNo, int pyBpNumber);
    void breakPointChanged(BreakPointItem oldBp, BreakPointItem newBp);
    ito::RetVal setupBreakPointDebugConnections();
    ito::RetVal shutdownBreakPointDebugConnections();
 
    bool renameVariable(bool globalNotLocal, const QString &oldFullItemName, QString newKey, ItomSharedSemaphore *semaphore = NULL);
    bool deleteVariable(bool globalNotLocal, const QStringList &fullItemNames);
    ito::RetVal pickleVariables(bool globalNotLocal, QString filename, QStringList varNames, ItomSharedSemaphore *semaphore = NULL);
    ito::RetVal pickleSingleParam(QString filename, QSharedPointer<ito::Param> value, const QString &valueName, ItomSharedSemaphore *semaphore = NULL);
    ito::RetVal unpickleVariables(bool globalNotLocal, QString filename, QString packedVarName, ItomSharedSemaphore *semaphore = NULL);
    ito::RetVal saveMatlabVariables(bool globalNotLocal, QString filename, QStringList varNames, ItomSharedSemaphore *semaphore = NULL);
    ito::RetVal saveMatlabSingleParam(QString filename, QSharedPointer<ito::Param> value, const QString &valueName, ItomSharedSemaphore *semaphore = NULL);
    ito::RetVal loadMatlabVariables(bool globalNotLocal, QString filename, QString packedVarName, ItomSharedSemaphore *semaphore = NULL);
    ito::RetVal registerAddInInstance(QString varname, ito::AddInBase *instance, ItomSharedSemaphore *semaphore = NULL);
    ito::RetVal getSysModules(QSharedPointer<QStringList> modNames, QSharedPointer<QStringList> modFilenames, QSharedPointer<IntList> modTypes, ItomSharedSemaphore *semaphore = NULL);
    ito::RetVal reloadSysModules(QSharedPointer<QStringList> modNames, ItomSharedSemaphore *semaphore = NULL);
 
    void registerWorkspaceContainer(PyWorkspaceContainer *container, bool registerNotUnregister, bool globalNotLocal);
    void workspaceGetChildNode(PyWorkspaceContainer *container, QString fullNameParentItem);
    void workspaceGetValueInformation(PyWorkspaceContainer *container, const QString &fullItemName, QSharedPointer<QString> extendedValue, ItomSharedSemaphore *semaphore = NULL);
 
    ito::RetVal checkVarnamesInWorkspace(bool globalNotLocal, const QStringList &names, QSharedPointer<IntList> existing, ItomSharedSemaphore *semaphore = NULL); 
    ito::RetVal putParamsToWorkspace(bool globalNotLocal, const QStringList &names, const QVector<SharedParamBasePointer > &values, ItomSharedSemaphore *semaphore = NULL);
    ito::RetVal getVarnamesListInWorkspace(bool globalNotLocal, const QString &find, QSharedPointer<QStringList> varnameList, ItomSharedSemaphore *semaphore /*= NULL*/);
    ito::RetVal getParamsFromWorkspace(bool globalNotLocal, const QStringList &names, QVector<int> paramBaseTypes, QSharedPointer<SharedParamBasePointerVector > values, ItomSharedSemaphore *semaphore = NULL);
 
    ito::RetVal pythonGetClearAllValues();
    ito::RetVal pythonClearAll();
 
};
 
} //end namespace ito