Repository: kwindrem/SetupHelper Branch: main Commit: d9ded5decd3e Files: 68 Total size: 700.4 KB Directory structure: gitextract_d5l_5hwy/ ├── .github/ │ └── workflows/ │ └── latest-tag.yml ├── FileSets/ │ ├── PatchSource/ │ │ ├── PageSettings.qml │ │ ├── PageSettings.qml.orig │ │ └── PageSettings.qml.patch │ ├── VersionIndependent/ │ │ ├── MbDisplayDefaultPackage.qml │ │ ├── MbDisplayPackageVersion.qml │ │ ├── PageSettingsAddPackageList.qml │ │ ├── PageSettingsPackageAdd.qml │ │ ├── PageSettingsPackageEdit.qml │ │ ├── PageSettingsPackageManager.qml │ │ ├── PageSettingsPackageVersions.qml │ │ ├── PageSettingsPmBackup.qml │ │ └── PageSettingsPmInitialize.qml │ ├── fileListPatched │ └── fileListVersionIndependent ├── HelperResources/ │ ├── CommonResources │ ├── DbusSettingsResources │ ├── EssentialResources │ ├── IncludeHelpers │ ├── LogHandler │ ├── ServiceResources │ └── VersionResources ├── PackageDevelopmentGuidelines.md ├── PackageManager.py ├── ReadMe.md ├── blindInstall/ │ ├── SetupHelperVersion │ ├── blindInstall.sh │ ├── post-hook.sh │ ├── pre-hook.sh │ └── rcS.localForUninstall ├── changes ├── defaultPackageList ├── forSetupScript ├── genericSetupScript ├── gitHubInfo ├── makeVelib_python ├── patch ├── patchBookworm ├── rcS.local ├── reinstallMods ├── services/ │ └── PackageManager/ │ ├── log/ │ │ └── run │ └── run ├── settingsList ├── setup ├── updatePackage ├── velib_python/ │ ├── dbusmonitor.py │ ├── oldestVersion │ ├── settingsdevice.py │ ├── ve_utils.py │ ├── vedbus.py │ └── velib_python/ │ ├── latest/ │ │ ├── dbusmonitor.py │ │ ├── oldestVersion │ │ ├── settingsdevice.py │ │ ├── ve_utils.py │ │ └── vedbus.py │ ├── v3.34/ │ │ ├── dbusmonitor.py │ │ ├── oldestVersion │ │ ├── settingsdevice.py │ │ ├── ve_utils.py │ │ └── vedbus.py │ └── v3.41/ │ ├── dbusmonitor.py │ ├── oldestVersion │ ├── settingsdevice.py │ ├── ve_utils.py │ └── vedbus.py ├── venus-data-SetupHelperInstall.tgz ├── venus-data-UninstallAllPackages.tgz └── version ================================================ FILE CONTENTS ================================================ ================================================ FILE: .github/workflows/latest-tag.yml ================================================ name: Add latest tag to new release on: release: types: [published] workflow_dispatch: jobs: run: name: Run local action runs-on: ubuntu-latest steps: - name: Checkout repository uses: actions/checkout@master - name: Run latest-tag uses: EndBug/latest-tag@v1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} ================================================ FILE: FileSets/PatchSource/PageSettings.qml ================================================ import QtQuick 1.1 import com.victron.velib 1.0 import net.connman 0.1 import "utils.js" as Utils MbPage { title: qsTr("Settings") property string bindPrefix: "com.victronenergy.settings" property VBusItem relay0Item: VBusItem {bind: "com.victronenergy.system/Relay/0/State"} property bool hasRelay0: relay0Item.valid model: VisibleItemModel { MbSubMenu { id: generalItem description: qsTr("General") subpage: Component { PageSettingsGeneral { title: generalItem.description } } } MbSubMenu { description: qsTr("Firmware") subpage: Component { PageSettingsFirmware { title: qsTr("Firmware") } } } MbSubMenu { description: qsTr("Date & Time") subpage: Component { PageTzInfo { title: qsTr("Date & Time") } } } MbSubMenu { description: qsTr("Remote Console") subpage: Component { PageSettingsRemoteConsole {} } } MbSubMenu { id: systemSetupItem description: qsTr("System setup") subpage: Component { PageSettingsSystem { title: systemSetupItem.description } } } MbSubMenu { id: dvcc description: qsTr("DVCC") subpage: Component { PageSettingsDVCC { title: dvcc.description } } } MbSubMenu { id: displayItem description: qsTr("Display & language") subpage: Component { PageSettingsDisplay { title: displayItem.description } } } MbSubMenu { id: vrmLoggerItem description: qsTr("VRM online portal") subpage: Component { PageSettingsLogger { title: vrmLoggerItem.description } } } MbSubMenu { VBusItem { id: systemType bind: "com.victronenergy.system/SystemType" } description: systemType.value === "Hub-4" ? systemType.value : qsTr("ESS") subpage: Component { PageSettingsHub4 {} } } MbSubMenu { description: qsTr("Energy meters") subpage: Component { PageSettingsCGwacsOverview {} } } MbSubMenu { description: qsTr("PV inverters") subpage: Component { PageSettingsFronius {} } } MbSubMenu { show: App.withQwacs description: qsTr("Wireless AC sensors") subpage: Component { PageSettingsQwacs {} } } MbSubMenu { description: qsTr("Modbus TCP/UDP devices") subpage: Component { PageSettingsModbus {} } } MbSubMenu { id: ethernetItem description: qsTr("Ethernet") subpage: Component { PageSettingsTcpIp { showLinkLocal: true } } } MbSubMenu { description: qsTr("Wi-Fi") property VeQuickItem accessPoint: VeQuickItem { uid: "dbus/com.victronenergy.platform/Services/AccessPoint/Enabled" } subpage: accessPoint.value !== undefined ? wifiWithAP : wifiWithoutAP Component { id: wifiWithoutAP; PageSettingsWifi {} } Component { id: wifiWithAP; PageSettingsWifiWithAccessPoint {} } } MbSubMenu { description: qsTr("GSM modem") subpage: Component { PageSettingsGsm {} } } MbSubMenu { description: qsTr("Bluetooth") subpage: Component { PageSettingsBluetooth {} } show: Connman.technologyList.indexOf("bluetooth") !== -1 } MbSubMenu { description: qsTr("GPS") subpage: Component { PageSettingsGpsList {} } } MbSubMenu { description: qsTr("Generator start/stop") subpage: Component { PageRelayGenerator {} } show: hasRelay0 } MbSubMenu { description: qsTr("Tank pump") subpage: Component { PageSettingsTankPump {} } } MbSubMenu { description: qsTr("Relay") subpage: Component { PageSettingsRelay {} } show: hasRelay0 } MbSubMenu { description: qsTr("Services") subpage: Component { PageSettingsServices {} } } MbSubMenu { description: qsTr("I/O") subpage: ioSettings show: ioSettings.haveSubMenus PageSettingsIo { id: ioSettings } } /* MbSubMenu { description: qsTr("Backup & Restore") subpage: Component { PageSettingsBackup {} } } */ MbSubMenu { description: qsTr("Venus OS Large features") subpage: Component { PageSettingsLarge {} } property VBusItem signalK: VBusItem { bind: "com.victronenergy.platform/Services/SignalK/Enabled" } property VBusItem nodeRed: VBusItem { bind: "com.victronenergy.platform/Services/NodeRed/Mode" } show: signalK.valid || nodeRed.valid } MbSubMenu { description: "Debug" subpage: Component { PageDebug {} } showAccessLevel: User.AccessService } //////// added for PackageManager MbSubMenu { description: qsTr("Package manager") subpage: Component { PageSettingsPackageManager {} } } } } ================================================ FILE: FileSets/PatchSource/PageSettings.qml.orig ================================================ import QtQuick 1.1 import com.victron.velib 1.0 import net.connman 0.1 import "utils.js" as Utils MbPage { title: qsTr("Settings") property string bindPrefix: "com.victronenergy.settings" property VBusItem relay0Item: VBusItem {bind: "com.victronenergy.system/Relay/0/State"} property bool hasRelay0: relay0Item.valid model: VisibleItemModel { MbSubMenu { id: generalItem description: qsTr("General") subpage: Component { PageSettingsGeneral { title: generalItem.description } } } MbSubMenu { description: qsTr("Firmware") subpage: Component { PageSettingsFirmware { title: qsTr("Firmware") } } } MbSubMenu { description: qsTr("Date & Time") subpage: Component { PageTzInfo { title: qsTr("Date & Time") } } } MbSubMenu { description: qsTr("Remote Console") subpage: Component { PageSettingsRemoteConsole {} } } MbSubMenu { id: systemSetupItem description: qsTr("System setup") subpage: Component { PageSettingsSystem { title: systemSetupItem.description } } } MbSubMenu { id: dvcc description: qsTr("DVCC") subpage: Component { PageSettingsDVCC { title: dvcc.description } } } MbSubMenu { id: displayItem description: qsTr("Display & language") subpage: Component { PageSettingsDisplay { title: displayItem.description } } } MbSubMenu { id: vrmLoggerItem description: qsTr("VRM online portal") subpage: Component { PageSettingsLogger { title: vrmLoggerItem.description } } } MbSubMenu { VBusItem { id: systemType bind: "com.victronenergy.system/SystemType" } description: systemType.value === "Hub-4" ? systemType.value : qsTr("ESS") subpage: Component { PageSettingsHub4 {} } } MbSubMenu { description: qsTr("Energy meters") subpage: Component { PageSettingsCGwacsOverview {} } } MbSubMenu { description: qsTr("PV inverters") subpage: Component { PageSettingsFronius {} } } MbSubMenu { show: App.withQwacs description: qsTr("Wireless AC sensors") subpage: Component { PageSettingsQwacs {} } } MbSubMenu { description: qsTr("Modbus TCP/UDP devices") subpage: Component { PageSettingsModbus {} } } MbSubMenu { id: ethernetItem description: qsTr("Ethernet") subpage: Component { PageSettingsTcpIp { showLinkLocal: true } } } MbSubMenu { description: qsTr("Wi-Fi") property VeQuickItem accessPoint: VeQuickItem { uid: "dbus/com.victronenergy.platform/Services/AccessPoint/Enabled" } subpage: accessPoint.value !== undefined ? wifiWithAP : wifiWithoutAP Component { id: wifiWithoutAP; PageSettingsWifi {} } Component { id: wifiWithAP; PageSettingsWifiWithAccessPoint {} } } MbSubMenu { description: qsTr("GSM modem") subpage: Component { PageSettingsGsm {} } } MbSubMenu { description: qsTr("Bluetooth") subpage: Component { PageSettingsBluetooth {} } show: Connman.technologyList.indexOf("bluetooth") !== -1 } MbSubMenu { description: qsTr("GPS") subpage: Component { PageSettingsGpsList {} } } MbSubMenu { description: qsTr("Generator start/stop") subpage: Component { PageRelayGenerator {} } show: hasRelay0 } MbSubMenu { description: qsTr("Tank pump") subpage: Component { PageSettingsTankPump {} } } MbSubMenu { description: qsTr("Relay") subpage: Component { PageSettingsRelay {} } show: hasRelay0 } MbSubMenu { description: qsTr("Services") subpage: Component { PageSettingsServices {} } } MbSubMenu { description: qsTr("I/O") subpage: ioSettings show: ioSettings.haveSubMenus PageSettingsIo { id: ioSettings } } /* MbSubMenu { description: qsTr("Backup & Restore") subpage: Component { PageSettingsBackup {} } } */ MbSubMenu { description: qsTr("Venus OS Large features") subpage: Component { PageSettingsLarge {} } property VBusItem signalK: VBusItem { bind: "com.victronenergy.platform/Services/SignalK/Enabled" } property VBusItem nodeRed: VBusItem { bind: "com.victronenergy.platform/Services/NodeRed/Mode" } show: signalK.valid || nodeRed.valid } MbSubMenu { description: "Debug" subpage: Component { PageDebug {} } showAccessLevel: User.AccessService } } } ================================================ FILE: FileSets/PatchSource/PageSettings.qml.patch ================================================ --- /Users/Kevin/GitHub/SetupHelper.copy/FileSets/PatchSource/PageSettings.qml.orig 2024-05-15 13:06:53 +++ /Users/Kevin/GitHub/SetupHelper.copy/FileSets/PatchSource/PageSettings.qml 2025-01-24 22:39:59 @@ -192,5 +192,11 @@ subpage: Component { PageDebug {} } showAccessLevel: User.AccessService } +//////// added for PackageManager + MbSubMenu + { + description: qsTr("Package manager") + subpage: Component { PageSettingsPackageManager {} } + } } } ================================================ FILE: FileSets/VersionIndependent/MbDisplayDefaultPackage.qml ================================================ //////// new for PackageManager import QtQuick 1.1 import com.victron.velib 1.0 import "utils.js" as Utils MbItem { id: root property int defaultIndex property string servicePrefix property bool isCurrentItem: root.ListView.isCurrentItem property MbStyle style: MbStyle { isCurrentItem: root.ListView.isCurrentItem } VBusItem { id: packageName; bind: getServiceBind ("PackageName") } onClicked: pageStack.push ("/opt/victronenergy/gui/qml/PageSettingsPackageAdd.qml", {defaultIndex: defaultIndex}) function getServiceBind(param) { return Utils.path(servicePrefix, "/Default/", defaultIndex, "/", param) } MbRowSmall { description: "" anchors.verticalCenter: parent.verticalCenter Column { width: root.width - gitHubUser.width - gitHubBranch.width - 20 Text // puts a bit of space above package name { text: " " font.pixelSize: 6 } Text { text:packageName.valid ? packageName.value : "" color: isCurrentItem ? root.style.textColorSelected : root.style.textColor font.pixelSize: 14 horizontalAlignment: Text.AlignLeft } Text { text: "" font.pixelSize: 10 horizontalAlignment: Text.AlignLeft } } Column { Text // puts a bit of space above version boxes { text: " " font.pixelSize: 3 } Text { text: "GitHub User" color: isCurrentItem ? root.style.textColorSelected : root.style.textColor font.pixelSize: 10 } MbTextBlock { id: gitHubUser item { bind: getServiceBind("GitHubUser") } height: 20; width: 120 } Text // puts a bit of space below version boxes - only needed in one column { text: " " font.pixelSize: 6 } } Column { Text // puts a bit of space above version boxes { text: " " font.pixelSize: 3 } Text { text: qsTr ("GitHub Tag") color: isCurrentItem ? root.style.textColorSelected : root.style.textColor font.pixelSize: 10 } MbTextBlock { id: gitHubBranch item { bind: getServiceBind("GitHubBranch") } height: 20; width: 120 } } } } ================================================ FILE: FileSets/VersionIndependent/MbDisplayPackageVersion.qml ================================================ //////// new for PackageManager import QtQuick 1.1 import com.victron.velib 1.0 import "utils.js" as Utils MbItem { id: root property int packageIndex property string servicePrefix property string settingsPrefix property bool isCurrentItem: root.ListView.isCurrentItem property MbStyle style: MbStyle { isCurrentItem: root.ListView.isCurrentItem } VBusItem { id: packageName; bind: getSettingsBind ("PackageName") } property VBusItem rebootNeededItem: VBusItem { bind: getServiceBind ( "RebootNeeded") } property VBusItem guiRestartNeededItem: VBusItem { bind: getServiceBind ( "GuiRestartNeeded") } property bool rebootNeeded: rebootNeededItem.valid && rebootNeededItem.value == 1 property bool guiRestartNeeded: guiRestartNeededItem.valid && guiRestartNeededItem.value == 1 VBusItem { id: incompatibleItem; bind: getServiceBind ( "Incompatible" ) } property string incompatibleReason: incompatibleItem.valid ? incompatibleItem.value : "" property bool compatible: incompatibleReason == "" VBusItem { id: platformItem; bind: Utils.path("com.victronenergy.packageManager", "/Platform" ) } property string platform: platformItem.valid ? platformItem.value : "???" // version info may be in platform service or in vePlatform.version VBusItem { id: osVersionItem; bind: Utils.path("com.victronenergy.platform", "/Firmware/Installed/Version" ) } property string osVersion: osVersionItem.valid ? osVersionItem.value : vePlatform.version onClicked: pageStack.push ("/opt/victronenergy/gui/qml/PageSettingsPackageEdit.qml", {newPackageIndex: packageIndex}) function statusText () { if (rebootNeeded) return qsTr ("REBOOT needed") if (guiRestartNeeded) return qsTr ("GUI restart needed") else if (incompatibleReason == 'PLATFORM') return qsTr ( "incompatible with " + platform ) // don't show warning incompatibilities here - they are shown in the editor menu else if (incompatibleReason != "" && incompatibleReason.toLowerCase().indexOf ("warning") == -1 ) return incompatibleReason else return "" } function getSettingsBind(param) { return Utils.path(settingsPrefix, "/", packageIndex, "/", param) } function getServiceBind(param) { return Utils.path(servicePrefix, "/Package/", packageIndex, "/", param) } MbRowSmall { description: "" anchors.verticalCenter: parent.verticalCenter Column { width: root.width - gitHubVersion.width - packageVersion.width - installedVersion.width - 20 Text // puts a bit of space above package name { text: " " font.pixelSize: 6 } Text { text:packageName.valid ? packageName.value : "" color: isCurrentItem ? root.style.textColorSelected : root.style.textColor font.pixelSize: 14 horizontalAlignment: Text.AlignLeft } Text { text: statusText () color: isCurrentItem ? root.style.textColorSelected : root.style.textColor font.pixelSize: 10 horizontalAlignment: Text.AlignLeft } } Column { Text // puts a bit of space above version boxes { text: " " font.pixelSize: 3 } Text { text: "GitHub" color: isCurrentItem ? root.style.textColorSelected : root.style.textColor font.pixelSize: 10 } MbTextBlock { id: gitHubVersion item { bind: getServiceBind("GitHubVersion") } height: 20; width: 99 } Text // puts a bit of space below version boxes - only needed in one column { text: " " font.pixelSize: 6 } } Column { Text // puts a bit of space above version boxes { text: " " font.pixelSize: 3 } Text { text: qsTr ("Stored") color: isCurrentItem ? root.style.textColorSelected : root.style.textColor font.pixelSize: 10 } MbTextBlock { id: packageVersion item { bind: getServiceBind("PackageVersion") } height: 20; width: 99 } } Column { Text // puts a bit of space above version boxes { text: " " font.pixelSize: 3 } Text { text: qsTr ("Installed") color: isCurrentItem ? root.style.textColorSelected : root.style.textColor font.pixelSize: 10 } MbTextBlock { id: installedVersion item { bind: getServiceBind("InstalledVersion") } height: 20; width: 99 } } } } ================================================ FILE: FileSets/VersionIndependent/PageSettingsAddPackageList.qml ================================================ /////// new menu for package version display import QtQuick 1.1 import "utils.js" as Utils import com.victron.velib 1.0 MbPage { id: root title: defaultCount.valid ? qsTr("Inactive packages (tap to activate) ") : qsTr ("Package manager not running") property string servicePrefix: "com.victronenergy.packageManager" // use DefaultCount as an indication that PackageManager is running property VBusItem defaultCount: VBusItem { bind: Utils.path(servicePrefix, "/DefaultCount") } model: defaultCount.valid ? defaultCount.value : 0 delegate: Component { MbDisplayDefaultPackage { servicePrefix: root.servicePrefix defaultIndex: index } } } ================================================ FILE: FileSets/VersionIndependent/PageSettingsPackageAdd.qml ================================================ /////// new menu for package add edit import QtQuick 1.1 import "utils.js" as Utils import com.victron.velib 1.0 MbPage { id: root title: editActionItem.valid ? qsTr("Add package") : qsTr ("Package manager not running") property bool isCurrentItem: root.ListView.isCurrentItem property MbStyle style: MbStyle { isCurrentItem: root.ListView.isCurrentItem } property string settingsPrefix: "com.victronenergy.settings/Settings/PackageManager" property string servicePrefix: "com.victronenergy.packageManager" property int defaultIndex:0 property VBusItem defaultCount: VBusItem { bind: Utils.path(servicePrefix, "/DefaultCount") } property VBusItem editActionItem: VBusItem { bind: Utils.path(servicePrefix, "/GuiEditAction") } property VBusItem editStatus: VBusItem { bind: Utils.path(servicePrefix, "/GuiEditStatus") } property string packageName: packageNameBox.item.valid ? packageNameBox.item.value : "" property string editAction: editActionItem.valid ? editActionItem.value : '' property VBusItem defaultPackageName: VBusItem { bind: Utils.path ( servicePrefix, "/Default/", defaultIndex, "/", "PackageName" ) } property VBusItem defaultGitHubUser: VBusItem { bind: Utils.path ( servicePrefix, "/Default/", defaultIndex, "/", "GitHubUser" ) } property VBusItem defaultGitHubBranch: VBusItem { bind: Utils.path ( servicePrefix, "/Default/", defaultIndex, "/", "GitHubBranch" ) } property VBusItem editPackageName: VBusItem { bind: Utils.path ( settingsPrefix, "/Edit/", "PackageName" ) } property VBusItem editGitHubUser: VBusItem { bind: Utils.path ( settingsPrefix, "/Edit/", "GitHubUser" ) } property VBusItem editGitHubBranch: VBusItem { bind: Utils.path ( settingsPrefix, "/Edit/", "GitHubBranch" ) } property bool addPending: false property bool entryValid: editPackageName.value != "" && editGitHubUser.value != "" && editGitHubBranch.value != "" Component.onCompleted: { updateEdit () } onEditActionChanged: { if (addPending && editAction == '') { addPending = false pageStack.pop() } } function getSettingsBind(param) { return Utils.path(settingsPrefix, "/Edit/", param) } function getServiceBind(param) { return Utils.path(servicePrefix, "/Default/", defaultIndex, "/", param) } // copy a set of default package values to Edit area when changing indexes function updateEdit () { bindPrefix = Utils.path(servicePrefix, "/Default/", defaultIndex ) var defaultName = defaultPackageName.valid ? defaultPackageName.value : "??" if (defaultName == "new") defaultName = "" editPackageName.setValue ( defaultName ) editGitHubUser.setValue ( defaultGitHubUser.valid ? defaultGitHubUser.value : "??" ) editGitHubBranch.setValue ( defaultGitHubBranch.valid ? defaultGitHubBranch.value : "??" ) editStatus.setValue ("") editActionItem.setValue ("") addPending = false } function cancelEdit () { addPending = false if (editAction == '') pageStack.pop() else { editStatus.setValue ("") editActionItem.setValue ("") } } function confirm () { if (entryValid) { addPending = true // provide local confirmation of action - takes PackageManager too long editStatus.setValue ( "adding " + packageName) editActionItem.setValue ('add:' + packageName) } } model: VisibleItemModel { MbEditBox { id: packageNameBox description: qsTr ("Package name") maximumLength: 30 item.bind: getSettingsBind ("PackageName") overwriteMode: false writeAccessLevel: User.AccessInstaller } MbEditBox { id: gitHubUserBox description: qsTr ("GitHub user") maximumLength: 20 item.bind: getSettingsBind ("GitHubUser") overwriteMode: false writeAccessLevel: User.AccessInstaller } MbEditBox { id: gitHubBranchBox description: qsTr ("GitHub branch or tag") maximumLength: 20 item.bind: getSettingsBind ("GitHubBranch") overwriteMode: false writeAccessLevel: User.AccessInstaller } MbOK { id: cancelButton width: 90 anchors { right: parent.right } description: "" value: editAction == '' ? qsTr("Cancel") : qsTr("OK") onClicked: cancelEdit () } MbOK { id: proceedButton width: 100 anchors { right: cancelButton.left; bottom: cancelButton.bottom } description: "" value: qsTr ("Proceed") onClicked: confirm () show: editAction == '' && entryValid writeAccessLevel: User.AccessInstaller } Text { id: statusMessage width: 250 wrapMode: Text.WordWrap anchors { left: parent.left; leftMargin: 10; bottom: cancelButton.bottom; bottomMargin: 5 } font.pixelSize: 12 text: { if (editStatus.valid && editStatus.value != "") return editStatus.value else if (entryValid) return ("add " + packageName + " ?") else if (editPackageName.value == "") return ("enter a unique package name") else if (editGitHubUser.value == "") return ("enter GitHub user") else if (editGitHubBranch.value == "") return ("enter GitHub branch") else return ("") } color: isCurrentItem ? root.style.textColorSelected : root.style.textColor } } } ================================================ FILE: FileSets/VersionIndependent/PageSettingsPackageEdit.qml ================================================ /////// new menu for package version edit import QtQuick 1.1 import "utils.js" as Utils import com.victron.velib 1.0 MbPage { id: root title: platform.valid ? qsTr("Package editor") : qsTr ("Package manager not running") property bool isCurrentItem: root.ListView.isCurrentItem property MbStyle style: MbStyle { isCurrentItem: root.ListView.isCurrentItem } property string settingsPrefix: "com.victronenergy.settings/Settings/PackageManager" property string servicePrefix: "com.victronenergy.packageManager" property int packageIndex: 0 property int newPackageIndex:0 property VBusItem packageCount: VBusItem { bind: Utils.path(settingsPrefix, "/Count") } property VBusItem editAction: VBusItem { bind: Utils.path(servicePrefix, "/GuiEditAction") } property VBusItem editStatus: VBusItem { bind: Utils.path(servicePrefix, "/GuiEditStatus") } property VBusItem packageNameItem: VBusItem { bind: getSettingsBind ("PackageName") } property string packageName: packageNameItem.valid ? packageNameItem.value : "" property bool isSetupHelper: packageName == "SetupHelper" property VBusItem incompatibleReasonItem: VBusItem { bind: getServiceBind ( "Incompatible" ) } property string incompatibleReason: incompatibleReasonItem.valid ? incompatibleReasonItem.value : "" property VBusItem incompatibleDetailsItem: VBusItem { bind: getServiceBind ( "IncompatibleDetails") } property string incompatibleDetails: incompatibleDetailsItem.valid ? incompatibleDetailsItem.value : "" property bool incompatible: incompatibleReason != "" property VBusItem platform: VBusItem { bind: Utils.path(servicePrefix, "/Platform") } property VBusItem incompatibleResolvableItem: VBusItem { bind: getServiceBind ( "IncompatibleResolvable") } property bool gitHubValid: gitHubVersion.item.valid && gitHubVersion.item.value.substring (0,1) === "v" property bool packageValid: packageVersion.item.valid && packageVersion.item.value.substring (0,1) === "v" property bool installedValid: installedVersion.item.valid && installedVersion.item.value.substring (0,1) === "v" property bool downloadOk: gitHubValid && gitHubVersion.item.value != "" property bool installOk: ! incompatible property string requestedAction: '' property bool actionPending: requestedAction != '' property bool waitForAction: editAction.value != '' && ! editError property bool editError: editAction.value == 'ERROR' property bool navigate: ! actionPending && ! waitForAction property bool detailsExist: incompatibleDetails != "" property bool detailsResolvable: incompatibleResolvableItem.valid ? incompatibleResolvableItem.value : "" property bool showDetails: false property string localError: "" // version info may be in platform service or in vePlatform.version VBusItem { id: osVersionItem; bind: Utils.path("com.victronenergy.platform", "/Firmware/Installed/Version" ) } property string osVersion: osVersionItem.valid ? osVersionItem.value : vePlatform.version // ActionNeeded is a global parameter provided inform the GUI that a GUI restart or system reboot is needed // when dismissed, a timer started which hides the information // when the timer expires, the information is shown again // changes to ActionNeeded will stop the timer so the new value will be shown immediately property VBusItem actionNeededItem: VBusItem { bind: Utils.path(servicePrefix, "/ActionNeeded") } property string actionNeeded: actionNeededItem.valid ? actionNeededItem.value : "" property bool showActionNeeded: ! hideActionNeededTimer.running && actionNeeded != '' onActionNeededChanged: { hideActionNeededTimer.stop () } onWaitForActionChanged: { if ( ! waitForAction ) { hideActionNeededTimer.stop () requestedAction = '' } } onIncompatibleChanged: { if (! incompatible ) showDetails = false } onActiveChanged: { if (active) { hideActionNeededTimer.stop () resetPackageIndex () refreshGitHubVersions () acknowledgeError () requestedAction = '' } } onNavigateChanged: resetPackageIndex () // hide action for 10 minutes Timer { id: hideActionNeededTimer running: false repeat: false interval: 1000 * 60 * 10 } // refresh the GitHub version GitHub version age is greater than 30 seconds property bool waitForIndexChange: false property bool waitForNameChange: false onPackageIndexChanged: { waitForIndexChange = false } onPackageNameChanged: { waitForNameChange = false refreshGitHubVersions () } function refreshGitHubVersions () { if ( waitForIndexChange || waitForNameChange ) return else if (! active || editAction.value != "" || actionPending) return sendCommand ( 'gitHubScan' + ':' + packageName, false ) } // acknowledge error reported from PackageManager // and erase status message function acknowledgeError () { if (editError) { editAction.setValue ("") editStatus.setValue ("") } } function resetPackageIndex () { if (waitForAction) return if (newPackageIndex < 0) newPackageIndex = 0 else if (newPackageIndex >= packageCount.value) newPackageIndex = packageCount.value - 1 if (newPackageIndex != packageIndex) { waitForIndexChange = true waitForNameChange = true packageIndex = newPackageIndex requestedAction = '' showDetails = false } } function getSettingsBind(param) { return Utils.path(settingsPrefix, "/", packageIndex, "/", param) } function getServiceBind(param) { return Utils.path(servicePrefix, "/Package/", packageIndex, "/", param) } function sendCommand (command, updateEditStatus ) { if (editAction.value != "") localError = "command could not be sent (" + command + ")" else { if (updateEditStatus) editStatus.setValue ("sending " + command) editAction.setValue (command) } } // don't change packages if pending operation or waiting for completion function nextIndex () { if (editError) return newPackageIndex += 1 resetPackageIndex () } function previousIndex () { if (editError) return newPackageIndex -= 1 resetPackageIndex () } function cancelEdit () { // cancel any pending operation requestedAction = '' showDetails = false acknowledgeError () // if was showing action needed, hide that messge for now if (showActionNeeded) hideActionNeededTimer.start () } function confirm () { if (showDetails) { if (detailsResolvable) { sendCommand ( 'resolveConflicts:' + packageName, true ) showDetails = false } // trigger setup script prechecks else { sendCommand ( 'check:' + packageName, true ) showDetails = false } } else if (actionPending) sendCommand ( requestedAction + ':' + packageName, true ) else if (showActionNeeded) { if (actionNeeded.indexOf ( "REBOOT" ) != -1 ) sendCommand ( 'reboot', true ) else if (actionNeeded.indexOf ( "restart" ) != -1 ) sendCommand ( 'restartGui', true ) hideActionNeededTimer.start () } requestedAction = '' } function install () { if (navigate && installOk && ! editError) { requestedAction = 'install' showDetails = false } } function uninstall () { if (navigate && installedValid && ! editError) { requestedAction = 'uninstall' showDetails = false } } function gitHubDownload () { if (navigate && downloadOk && ! editError) { requestedAction = 'download' showDetails = false } } function remove () { if ( ! editError) { requestedAction = 'remove' showDetails = false } } model: VisibleItemModel { MbItemText { id: packageNameBox text: packageName + " versions" } Row { height: 25 spacing: 1 // leftPadding: 7; rightPadding: 5; can't use in earlier Qt versions // use dummy text field for left spacing instead Text { text: " " font.pixelSize: 10 } Text { text: "GitHub:" color: isCurrentItem ? root.style.textColorSelected : root.style.textColor font.pixelSize: 10 } MbTextBlock { id: gitHubVersion item { bind: getServiceBind("GitHubVersion") } height: 25; width: 112 } Text { text: qsTr (" stored:") color: isCurrentItem ? root.style.textColorSelected : root.style.textColor font.pixelSize: 10 } MbTextBlock { id: packageVersion item { bind: getServiceBind("PackageVersion") } height: 25; width: 112 } Text { text: qsTr (" installed:") color: isCurrentItem ? root.style.textColorSelected : root.style.textColor horizontalAlignment: Text.AlignRight font.pixelSize: 10 } MbTextBlock { id: installedVersion item { bind: getServiceBind("InstalledVersion") } height: 25 width: 112 } } MbEditBox { id: gitHubUser description: qsTr ("GitHub user") maximumLength: 20 item.bind: getSettingsBind ("GitHubUser") overwriteMode: false writeAccessLevel: User.AccessInstaller } MbEditBox { id: gitHubBranch description: qsTr ("GitHub branch or tag") maximumLength: 20 item.bind: getSettingsBind ("GitHubBranch") overwriteMode: false writeAccessLevel: User.AccessInstaller } MbOK { id: cancelButton width: 85 anchors { right: gitHubBranch.right; bottom: statusMessage.bottom } description: "" value: ( actionPending || showDetails ) ? qsTr("Cancel") : (editError ? qsTr("OK") : qsTr("Later")) onClicked: cancelEdit () show: ( actionPending || showDetails || editError || showActionNeeded ) && ! waitForAction } MbOK { id: confirmButton width: 92 anchors { right: cancelButton.left; bottom: statusMessage.bottom } description: "" value: ( actionPending || detailsResolvable ) ? qsTr("Proceed") : showDetails ? qsTr ("Recheck") : qsTr ("Now") onClicked: confirm () show: ( actionPending || showDetails || showActionNeeded ) && ! waitForAction writeAccessLevel: User.AccessInstaller } MbOK { id: showDetailsButton width: 150 anchors { right: gitHubBranch.right; bottom: statusMessage.bottom} description: "" value: qsTr("Show Details") onClicked: showDetails = true writeAccessLevel: User.AccessInstaller show: navigate && detailsExist && ! ( editError || actionPending || waitForAction || showActionNeeded || showDetails) } Text { id: statusMessage width: { var smWidth = root.width if (cancelButton.show) smWidth -= cancelButton.width if (confirmButton.show) smWidth -= confirmButton.width if (showDetailsButton.show) smWidth -= showDetailsButton.width return smWidth } height: Math.max (paintedHeight, 35) wrapMode: Text.WordWrap horizontalAlignment: Text.AlignLeft anchors { left: gitHubBranch.left; leftMargin: 5; top: gitHubBranch.bottom } font.pixelSize: 12 color: isSetupHelper && requestedAction == 'uninstall' ? "red" : root.style.textColor text: { if (showDetails) { if (detailsResolvable) return ( incompatibleDetails + qsTr ("\nResolve conflicts?") ) else return ( incompatibleDetails ) } else if (actionPending) { if (isSetupHelper && requestedAction == 'uninstall') return qsTr ("WARNING: SetupHelper is required for these menus - uninstall anyway ?") else return (requestedAction + " " + packageName + " ?") } else if (editStatus.valid && editStatus.value != "") return ( editStatus.value ) else if (showActionNeeded) return ( actionNeeded ) else if (incompatible) return ( incompatibleReason ) else return localError } } // bottom row of buttons MbOK { id: previousButton width: 100 anchors { left: gitHubBranch.left; top: statusMessage.bottom; topMargin: 5 } description: "" value: qsTr("Previous") onClicked: previousIndex () opacity: ! editError && newPackageIndex > 0 ? 1.0 : 0.2 } MbOK { id: nextButton width: 70 anchors { left: previousButton.right; top: statusMessage.bottom; topMargin: 5 } description: "" value: qsTr("Next") onClicked: nextIndex () opacity: ! editError && (newPackageIndex < packageCount.value - 1) ? 1.0 : 0.2 } MbOK { id: downloadButton width: 110 anchors { right: installButton.left; top: statusMessage.bottom; topMargin: 5 } description: "" value: qsTr ("Download") onClicked: gitHubDownload () opacity: ! editError && navigate && downloadOk > 0 ? 1.0 : 0.2 writeAccessLevel: User.AccessInstaller } MbOK { id: installButton width: 80 anchors { right: uninstallButton.left; top: statusMessage.bottom; topMargin: 5 } description: "" value: qsTr ("Install") onClicked: install () opacity: ! editError && navigate && installOk > 0 ? 1.0 : 0.2 writeAccessLevel: User.AccessInstaller } MbOK { id: uninstallButton width: 105 anchors { right: gitHubBranch.right; top: statusMessage.bottom; topMargin: 5 } description: "" value: installedValid ? qsTr("Uninstall") : qsTr("Remove") onClicked: installedValid ? uninstall () : remove () opacity: ! editError && navigate ? 1.0 : 0.2 writeAccessLevel: User.AccessInstaller } // dummy item to allow scrolling to show last button line when status message has many lines MbItemText { text: "" opacity: 0 show: statusMessage.height > 35 } } } ================================================ FILE: FileSets/VersionIndependent/PageSettingsPackageManager.qml ================================================ /////// new menu for package version display import QtQuick 1.1 import "utils.js" as Utils import com.victron.velib 1.0 MbPage { id: root title: showControls ? qsTr("Package manager") : qsTr("Package manager not running") property string settingsPrefix: "com.victronenergy.settings/Settings/PackageManager" property string servicePrefix: "com.victronenergy.packageManager" property string bindVrmloggerPrefix: "com.victronenergy.logger" VBusItem { id: pmStatusItem; bind: Utils.path(servicePrefix, "/PmStatus") } property string pmStatus: pmStatusItem.valid ? pmStatusItem.value : "" VBusItem { id: mediaStatus; bind: Utils.path(servicePrefix, "/MediaUpdateStatus") } VBusItem { id: actionNeeded; bind: Utils.path(servicePrefix, "/ActionNeeded") } VBusItem { id: editAction; bind: Utils.path(servicePrefix, "/GuiEditAction") } property bool showMediaStatus: mediaStatus.valid && mediaStatus.value != "" property bool showControls: pmStatusItem.valid // the last status message received from PackageManager is saved in lastStatus // so there is some status to display when PackageManager quits property string lastStatus: "" onPmStatusChanged: { if (pmStatusItem.valid) lastStatus = pmStatus } model: VisibleItemModel { MbItemText { id: status text: { if (mediaStatus.valid && mediaStatus.value != "") return mediaStatus.value else if (showControls) return pmStatusItem.value else return lastStatus } wrapMode: Text.WordWrap horizontalAlignment: Text.AlignHCenter } MbItemOptions { id: autoDownload description: qsTr ("GitHub check frequency") bind: Utils.path (settingsPrefix, "/GitHubAutoDownload") possibleValues: [ MbOption { description: "Once"; value: 99 }, MbOption { description: "Every 10 minutes"; value: 1 }, MbOption { description: "Hourly"; value: 2 }, MbOption { description: "Daily"; value: 3 }, MbOption { description: "Never"; value: 0 } ] writeAccessLevel: User.AccessInstaller } MbSwitch { id: autoInstall bind: Utils.path (settingsPrefix, "/AutoInstall") name: qsTr ("Auto install packages") writeAccessLevel: User.AccessInstaller } MbSubMenu { description: qsTr("Active packages") subpage: Component { PageSettingsPackageVersions {} } show: showControls } MbSubMenu { description: qsTr("Inactive packages") subpage: Component { PageSettingsAddPackageList {} } show: showControls } MbOK { id: finishButton description: { if (editAction.value == 'reboot') return qsTr ("REBOOTING ...") else if (editAction.value == 'guiRestart') return qsTr ("restarting GUI ...") else return qsTr ("action to finish install/uninstall") } value: { if (! actionNeeded.valid) return "" else if (actionNeeded.value.indexOf ( "REBOOT" ) != -1 ) return qsTr ("Reboot") else if (actionNeeded.value.indexOf ( "restart" ) != -1 ) return qsTr ("Restart GUI") else return "" } onClicked: { if (finishButton.value == 'REBOOT') { // needs immediate update because GUI will be going down ASAP finishButton.description = qsTr ("REBOOTING ...") editAction.setValue ( 'reboot' ) } else if (finishButton.value == 'guiRestart') { // needs immediate update because GUI will be going down ASAP finishButton.description = qsTr ("restarting GUI ...") editAction.setValue ( 'restartGui' ) } } show: actionNeeded.valid && actionNeeded.value != '' writeAccessLevel: User.AccessInstaller } MbSubMenu { description: qsTr("Backup & restore settings") subpage: Component { PageSettingsPmBackup {} } show: showControls } MbOK { property int notMounted: 0 property int mounted: 1 property int unmountRequested: 2 property int unmountBusy: 3 function mountStateToText(s) { switch (s) { case mounted: return qsTr("Press to eject"); case unmountRequested: case unmountBusy: return qsTr("Ejecting, please wait"); default: return qsTr("No storage found"); } } VBusItem { id: vMountState bind: Utils.path(bindVrmloggerPrefix, "/Storage/MountState") } description: qsTr("microSD / USB") value: mountStateToText(vMountState.value) writeAccessLevel: User.AccessUser onClicked: vMountState.setValue(unmountRequested); editable: vMountState.value === mounted cornerMark: false } MbSubMenu { description: qsTr("Restart or initialize ...") subpage: Component { PageSettingsPmInitialize {} } show: showControls } } } ================================================ FILE: FileSets/VersionIndependent/PageSettingsPackageVersions.qml ================================================ /////// new menu for package version display import QtQuick 1.1 import "utils.js" as Utils import com.victron.velib 1.0 MbPage { id: root title: defaultCount.valid ? qsTr("Active packages (tap to edit) ") : qsTr ("Package manager not running") property string servicePrefix: "com.victronenergy.packageManager" property string settingsPrefix: "com.victronenergy.settings/Settings/PackageManager" property VBusItem count: VBusItem { bind: Utils.path(settingsPrefix, "/Count") } // use DefaultCount as an indication that PackageManager is running property VBusItem defaultCount: VBusItem { bind: Utils.path(servicePrefix, "/DefaultCount") } property VBusItem editAction: VBusItem { bind: Utils.path(servicePrefix, "/GuiEditAction") } // notify PackageManager to refresh GitHub versions for all packages // when this menu goes active (entering from parent or returning from child) // or if first package's GitHub version age is greater than 60 seconds onActiveChanged: refreshGitHubVersions () function refreshGitHubVersions () { if (! active) return else if ( editAction.value != "" ) return editAction.setValue ('gitHubScan:ALL') } model: defaultCount.valid ? count.valid ? count.value : 0 : 0 delegate: Component { MbDisplayPackageVersion { servicePrefix: root.servicePrefix settingsPrefix: root.settingsPrefix packageIndex: index } } } ================================================ FILE: FileSets/VersionIndependent/PageSettingsPmBackup.qml ================================================ /////// new menu for settings backup and restore import QtQuick 1.1 import "utils.js" as Utils import com.victron.velib 1.0 MbPage { id: root title: qsTr("Settings backup & restore") property string settingsPrefix: "com.victronenergy.settings/Settings/PackageManager" property string servicePrefix: "com.victronenergy.packageManager" VBusItem { id: mediaAvailable; bind: Utils.path(servicePrefix, "/BackupMediaAvailable") } VBusItem { id: settingsFileExists; bind: Utils.path(servicePrefix, "/BackupSettingsFileExist") } VBusItem { id: settingsLocalFileExists; bind: Utils.path(servicePrefix, "/BackupSettingsLocalFileExist") } VBusItem { id: backupProgressItem; bind: Utils.path(servicePrefix, "/BackupProgress") } property int backupProgress: backupProgressItem.valid ? backupProgressItem.value : 0 model: VisibleItemModel { MbItemText { id: info text: qsTr ("Backup and restore\nSOME system settings, logs and logos\nthis is NOT the Victron mechanism\ncurrently under development") wrapMode: Text.WordWrap horizontalAlignment: Text.AlignHCenter } MbItemText { id: status text: { if (backupProgress == 21 || backupProgress == 23) return qsTr ("backing up settings to local storage ... (may take a while)") else if (backupProgress == 22 || backupProgress == 24) return qsTr ("restoring settings from local storage ... (may take a while)") else if (backupProgress == 1 || backupProgress == 3) return qsTr ("backing up settings ... (may take a while)") else if (backupProgress == 2 || backupProgress == 4) return qsTr ("restoring settings ... (may take a while)") else if ( ! mediaAvailable.valid || mediaAvailable.value == 0) return qsTr ("No USB or SD media found - insert one to continue") else if (settingsFileExists.valid && settingsFileExists.value == 1) return qsTr ("Settings backup file found") else return "" } wrapMode: Text.WordWrap horizontalAlignment: Text.AlignHCenter } MbOK { description: qsTr("Backup settings, logos, logs") value: qsTr("Press to backup") onClicked: backupProgressItem.setValue (1) show: mediaAvailable.valid && mediaAvailable.value == 1 && backupProgressItem.value == 0 writeAccessLevel: User.AccessInstaller } MbOK { description: qsTr("Restore settings, logos") value: qsTr("Press to restore") onClicked: backupProgressItem.setValue (2) show: settingsFileExists.valid && settingsFileExists.value == 1 && backupProgressItem.value == 0 writeAccessLevel: User.AccessInstaller } MbOK { description: qsTr("Backup settings to local storage") value: qsTr("Press to backup") onClicked: backupProgressItem.setValue (21) show: backupProgressItem.value == 0 writeAccessLevel: User.AccessInstaller } MbOK { description: qsTr("Restore settings to from storage") value: qsTr("Press to restore") onClicked: backupProgressItem.setValue (22) show: settingsLocalFileExists.valid && settingsLocalFileExists.value == 1 && backupProgressItem.value == 0 writeAccessLevel: User.AccessInstaller } } } ================================================ FILE: FileSets/VersionIndependent/PageSettingsPmInitialize.qml ================================================ /////// new menu for PackageManager initialize import QtQuick 1.1 import "utils.js" as Utils import com.victron.velib 1.0 MbPage { id: root title: pmRunning ? qsTr("PackageManager restart/initialize") : qsTr ("Package manager not running") property string settingsPrefix: "com.victronenergy.settings/Settings/PackageManager" VBusItem { id: pmStatus; bind: Utils.path(servicePrefix, "/PmStatus") } property bool pmRunning: pmStatus.valid property bool showInProgress: false property string initializeMessage: "" onPmRunningChanged: { showInProgress = false } function sendCommand (command, message) { initializeMessage = message showInProgress = true editAction.setValue (command) } model: VisibleItemModel { MbOK { description: qsTr("Restart") value: qsTr("Press to restart Package Manager") onClicked:sendCommand ("RESTART_PM", qsTr ("restarting Package Manager ...")) writeAccessLevel: User.AccessInstaller show: ! showInProgress } MbOK { description: qsTr("Restart GUI") value: qsTr("Press to restart GUI") onClicked:sendCommand ("restartGui", qsTr ("restarting GUI ...")) writeAccessLevel: User.AccessInstaller show: ! showInProgress } MbItemText { id: info text: qsTr ("Initializing PackageManager will\nreset persistent storage to an empty state\nGit Hub user and branch are reset to defaults\nPackages added manually must be added again") wrapMode: Text.WordWrap horizontalAlignment: Text.AlignHCenter show: ! showInProgress } MbOK { description: qsTr("Initialize") value: qsTr("Press to INITIALIZE Package Manager") onClicked: sendCommand ("INITIALIZE_PM", qsTr ("INITIALIZING Package Manager ...")) writeAccessLevel: User.AccessInstaller show: ! showInProgress } MbItemText { id: initializingMessage text: initializeMessage wrapMode: Text.WordWrap horizontalAlignment: Text.AlignHCenter show: showInProgress } } } ================================================ FILE: FileSets/fileListPatched ================================================ /opt/victronenergy/gui/qml/PageSettings.qml ================================================ FILE: FileSets/fileListVersionIndependent ================================================ /opt/victronenergy/gui/qml/PageSettingsPackageManager.qml /opt/victronenergy/gui/qml/PageSettingsPackageVersions.qml /opt/victronenergy/gui/qml/PageSettingsPackageEdit.qml /opt/victronenergy/gui/qml/PageSettingsAddPackageList.qml /opt/victronenergy/gui/qml/PageSettingsPackageAdd.qml /opt/victronenergy/gui/qml/PageSettingsPmBackup.qml /opt/victronenergy/gui/qml/PageSettingsPmInitialize.qml /opt/victronenergy/gui/qml/MbDisplayPackageVersion.qml /opt/victronenergy/gui/qml/MbDisplayDefaultPackage.qml ================================================ FILE: HelperResources/CommonResources ================================================ #!/bin/bash # CommonResources for SetupHelper # contains a functions and variables necessary for a setup script to interface with reinstallMods # # Refer to the SetupHelper ReadMe file for details on how to use these resources. # what action the script should take: # NONE - do noting - signals script to prompt for user input on how to proceed # INSTALL - install package components # (decommissioned) PROMPT - prompt user for additional installation options # UNINSTALL - remove package components # EXIT - exit script without taking any action # CHECK - runs file set checks only # this will attempt to create a missing file set so PackageManager # won't report it missing # CommonResources may set the the action if initial checks # indicate a clear direction # otherwise, the action will be set based on user input (in the script) # if failures occur during installation, # scriptAction should be changed to UNINSTALL so the installation can be cleaned up # and the setup script should test for UNINSTALL after it attempts installation # A file set error indicates the file set for the current verion is not usable # and installation should not occur # checkFileSets EXITS locally scriptAction='NONE' # flags to control setup script behavior (in endScript) rebootNeeded=false runAgain=false filesUpdated=false restartGui=false restartGeneratorService=false restartSystemCalc=false restartDigitalinputs=false # file lists are populated by getFileLists called from_chckFileSets and autoinstall # so these are global fileList=() fileListVersionIndependent=() fileListAll=() ######## skip to bottom of file for remainder of code executed when script is sourced # cleanup on any exit exitCleanup () { # remove temp directory rm -rf "$tempFileDir" } trap exitCleanup EXIT # checkPackageDependencies checks the packageDependencies file in the package directory # # all unmet dependencies are reported to the command line/log # and to stderr if not running from the command line # then the script exists function checkPackageDependencies () { dependencyFile="$scriptDir/packageDependencies" #no dependencies specified for this package if ! [ -f "$dependencyFile" ]; then return; fi errors="" while IFS= read -r line; do error="" read package requirement <<< "$line" if [ -f "$installedVersionPrefix"$package ]; then packageInstalled=true else packageInstalled=false fi case $requirement in installed) if ! $packageInstalled ; then error="$package must be installed" fi ;; uninstalled) if $packageInstalled ; then error="$package must be uninstalled" fi ;; esac if ! [ -z "$error" ]; then if ! [ -z "$errors" ]; then errors+=", " fi errors+="$error" fi done < "$dependencyFile" if ! [ -z "$errors" ]; then setInstallFailed $EXIT_PACKAGE_CONFLICT "$errors" fi } # getFileLists reads the file list from files in the FileSets directory # # 'fileList' file must only list version-dependent files # 'fileListVersionIndependent' file must list only version-independent files # prior to SetupHelper v6.0, this list is ignored # 'fileListPatched' lists all files that should be patched before installation # # $1 specifies where the path to the fileList files # # three composite file lists are returned in global arrays: # fileList contains only version-dependent files # fileListVersionIndependent contains only version-independent files # fileListPatched contains only files that need to be patched # fileListAll contains both versioned and version-independent files function getFileLists () { local verListFile="$1/fileList" local indListFile="$1/fileListVersionIndependent" local patchListFile="$1/fileListPatched" local tempListVer=() local tempListInd=() local tempListPatched=() if [ -f "$verListFile" ]; then while read -r line || [[ -n "$line" ]]; do read -a params <<< $line # parse line into space-separted parameters then discard any that don't begin with / # this strips all comments beginning with # as well as any leading or trailing spaces for param in ${params[@]} ; do case $param in /*) tempListVer+=("$param") ;; esac done done < "$verListFile" fi if [ -f "$indListFile" ]; then while read -r line || [[ -n "$line" ]]; do read -a params <<< $line for param in ${params[@]} ; do case $param in /*) tempListInd+=("$param") ;; esac done done < "$indListFile" fi if [ -f "$patchListFile" ]; then while read -r line || [[ -n "$line" ]]; do read -a params <<< $line for param in ${params[@]} ; do case $param in /*) tempListPatched+=("$param") ;; esac done done < "$patchListFile" fi # remove duplicate files from each list fileList=($(printf "%s\n" "${tempListVer[@]}" | sort -u)) fileListVersionIndependent=($(printf "%s\n" "${tempListInd[@]}" | sort -u)) fileListPatched=($(printf "%s\n" "${tempListPatched[@]}" | sort -u)) tempListAll=(${fileList[@]}) tempListAll+=(${fileListVersionIndependent[@]}) tempListAll+=(${fileListPatched[@]}) fileListAll=($(printf "%s\n" "${tempListAll[@]}" | sort -u)) } # yesNoPrompt provides user prompting requesting a yes/no response # # $1 is the prompt displayed when pausing for user input # # $yesResponse is set to true if the response was yes # # returns 0 for yes, 1 for no yesNoPrompt () { response='' while true; do /bin/echo -n "$*" read response case $response in [yY]*) yesResponse=true return 0 break ;; [nN]*) yesResponse=false return 1 break ;; *) esac done } # standardActionPrompt provides the standard set of options for selecting script's action # scriptAction is set by install/uninstall # other actions are handled locally, including quitting from the script # # if nonstandard prompts are necessary, duplicate this code in the setup script # and add the additional options and do not call standardActionPrompt # # the reinstall option is permitted only if setup options were previously set # if the the reinstall action is choosen, the script action is set to INSTALL # the setup script can then test this to skip further prompts # # $1 indicates if there are additional prompts needed during installaiton # if this parameter is 'MORE_PROMPTS', installaiton does NOT change scriptAction # if this parameter does not exist, installation WILL change scriptAction to INSTALL # this provides backaward compatibility with scripts written prior to the reinstall logic # standardActionPrompt () { if [ $# -gt 0 ] && [ $1 == 'MORE_PROMPTS' ]; then updateScriptAction=false else updateScriptAction=true fi echo echo "Available actions:" # don't allow install choice if incompatibilities have already been detected if ! $installFailed ; then echo " Install and activate (i)" else echo " can't install - errors reported above" fi if $optionsSet ; then echo " Reinstall (r) based on options provided at last install" fi echo " Uninstall (u) and restores all files to stock" echo " Quit (q) without further action" echo " Display log (l) outputs the last 100 lines of the log" echo response='' while true; do /bin/echo -n "Choose an action from the list above: " read response case $response in [iI]*) if ! $installFailed ; then if $updateScriptAction ; then scriptAction='INSTALL' fi break fi ;; [rR]*) if $optionsSet ; then scriptAction='INSTALL' break fi ;; [uU]*) scriptAction='UNINSTALL' break ;; [qQ]*) exit $EXIT_SUCCESS ;; [lL]*) tail -100 "$logFile" | tai64nlocal ;; *) esac done } # forcePackageUninstall insures a conflicting package is uninstalled before # this package is installed # the setup script must call this script BEFORE it begins installing anything # $1 is the package name # $2 contains an optional message forcePackageUninstall () { if (( $# < 1 )); then return fi if [ -f "$installedVersionPrefix""$1" ]; then if (( $# >= 2 )); then logMessage "${@:2}" else logMessage "uninstalling $1 - it conflicts with $packageName" fi if [ -e "/data/$1/setup" ]; then "/data/$1/setup" "uninstall" "auto" "deferReboot" "deferGuiRestart" else logMessage "WARNING can't uninstall $1 - no package directory or no setup script" fi if [ -e "/data/settupOptions/$1" ]; then touch "/data/settupOptions/$1/DO_NOT_AUTO_INSTALL" fi fi } # backupActiveFile makes a copy of the active file in file.orig # if the original file does not exist the NO_ORIG flag is set # to allow restoreAciveFile to remove the active file # # if the backup (.orig file) exists the backup is not updated # # $1 is the full path/file name to be backed up # # returns 0 if backup was made, 1 if not # backupActiveFile () { # don't do any work if install has already failed if $installFailed ; then return 1 fi local activeFile="$1" local origFile="$activeFile.orig" local noOrigFile="$activeFile.NO_ORIG" if [ -e "$activeFile" ]; then if ! [ -e "$origFile" ]; then cp "$activeFile" "$origFile" rm -f "$noOrigFile" return 0 else return 1 fi else if ! [ -e "$noOrigFile" ]; then touch "$noOrigFile" return 0 else return 1 fi fi } # SetupHelper maintains a set of "restart flags" # that control service restarts and system reboot # after the package has been installed / uninstalled # # some restarts/reboots are based on the directory of the modified file # others are based on the actual file itself # # not all services or specific files are flagged so some work in the setup script may be needed # # $1 is the full path and name to the modified file updateRestartFlags () { # flag indicating any file update occurred filesUpdated=true case $1 in /opt/victronenergy/gui*) restartGui=true return;; #### TODO: add gui-v2 /opt/victronenergy/dbus-generator*/*) restartGeneratorService=true return;; /opt/victronenergy/dbus-systemcalc-py/*) restartSystemCalc=true return;; /opt/victronenergy/dbus-digitalinputs/*) restartDigitalinputs=true return;; /u-boot/overlay./*) # Raspberry PI DT overlay rebootNeeded=true return;; /etc/udev/rules.d/*) # udev rules directory rebootNeeded=true return;; esac # reboots based on specific file case $( basename $1 ) in gpio_list) rebootNeeded=true return;; config.txt) rebootNeeded=true return;; esac } # updateActiveFile first backs up the active file # then copies the replacement (aka source) to the active file location (aka destination) # # two variations: # # updateActiveFile activeFile # an attempt is made to locate the source (replacement) # in the version directory or FileSets # # updateActiveFile sourceFile activeFile # a separate source (replacement) file is specified # # both sourceFile and activeFile must be a full path to the file # # if the update fails, scriptAction is changed to UNINSTALL # # global thisFileUpdated is set to true if file was updated, false if not # thisFileUpdated supports the old mechanism which may be used in some setup scripts # returns 0 if file was updated, 1 if not updateActiveFile () { thisFileUpdated=false # don't do any work if install has already failed if $installFailed ; then return 1 fi local sourceFile="" local activeFile="" # separate source and replacement files specified if [ $# == 2 ]; then if [ -f "$1" ]; then sourceFile="$1" else setInstallFailed $EXIT_FILE_SET_ERROR "specified soure file "$1" does not exist" return 1 fi activeFile="$2" # use active file for both source and destination else activeFile="$1" fi local baseName=$(basename "$activeFile") # replacement files are not needed for some versions # if so marked, leave original untouched if [ -e "$fileSet/$baseName.USE_ORIGINAL" ]; then return 1 fi # the location of the active file must exist if [ ! -e $(dirname "$activeFile") ]; then logMessage "path to $activeFile does not exist - skiping update" return 1 fi local usePatchedFile=false local patchedReplacement="$tempFileDir/$baseName.patchedForInstall" local currentPatchFile="$tempFileDir/$baseName.currentPatch" # source file not specified separately - look for it in expected places if [ -z "$sourceFile" ]; then # first in temp files - patched file if [ -e "$patchedReplacement" ] && [ -e "$currentPatchFile" ]; then sourceFile="$patchedReplacement" usePatchedFile=true # then in temp files - replacement elif [ -e "$tempFileDir/$baseName" ]; then sourceFile="$tempFileDir/$baseName" # then in version-specific FileSet elif [ -e "$fileSet/$baseName" ]; then sourceFile="$fileSet/$baseName" # then in version-independent file set elif [ -e "$versionIndependentFileSet/$baseName" ]; then sourceFile="$versionIndependentFileSet/$baseName" # then in FileSets (previous location of version-indepencent files) elif [ -e "$pkgFileSets/$baseName" ]; then sourceFile="$pkgFileSets/$baseName" else # if directory for file exists but no sourceFile - can't continue if ! [ -e $( dirname "$activeFile" ) ]; then logMessage "enclosing directory for $activeFile not found - skipping update" elif ! [ -e "$activeFile" ]; then logMessage "no replacement or active file $activeFile - skipping update" else setInstallFailed $EXIT_FILE_SET_ERROR "no soure file for $activeFile" fi return 1 fi fi # can't continue if other packages modified this file and this is a replacement (not a patch) local local packageList="$activeFile.package" local previousPackage="" local matchFound=false if [ -e "$packageList" ]; then previousPackages=$( cat "$packageList" ) for previousPackage in ${previousPackages[@]}; do if [ "$packageName" == "$previousPackage" ]; then matchFound=true elif ! $usePatchedFile ; then setInstallFailed $EXIT_PACKAGE_CONFLICT "$baseName was already modfied by $previousPackage" return 1 fi done fi # add file to installed files list (used by uninstallAll) if ! [ -e "$installedFilesList" ] \ || (( $( grep -c "$activeFile" "$installedFilesList" ) == 0 )); then echo "$activeFile" >> "$installedFilesList" fi # save the current patch file for use during a future uninstall local previousPatchFile="$previousPatchesDir/$baseName.patch" if $usePatchedFile ; then cp "$currentPatchFile" "$previousPatchFile" # no patch file used for this update else rm -f "$previousPatchFile" fi # if replacement, replace the .package file if ! $usePatchedFile ; then echo $packageName > "$packageList" # if patch and this package not in list yet, add add it elif ! $matchFound ; then echo $packageName >> "$packageList" fi # update the active file if needed # patched files have already incorporated current active file content # so nothing diffrerent here if ! [ -f "$activeFile" ] || ! cmp -s "$sourceFile" "$activeFile" ; then backupActiveFile "$activeFile" cp "$sourceFile" "$activeFile" updateRestartFlags "$activeFile" thisFileUpdated=true fi # insure active file has proper permissions if [ -f "$activeFile" ]; then chmod +r "$activeFile" if [[ -x "$activeFile.orig" ]] || [[ -x "$sourceFile" ]] ; then chmod +x "$activeFile" fi fi if $thisFileUpdated; then return 0 else return 1 fi } # end updateActiveFile () # restoreActiveFile # restores the active file to the content before this package modified it # for replacements, the backup copy is moved to the active location # if the backup copy doesn't exist BUT the NO_ORIG flag is set # the active copy is deleted to restore the system to stock # # for patches, this package's changes are removed from the active file # by reverse patching it # # if the reverse patch fails, the original is restored # modifications from all other packages are also removed !! # this is drastic but has best chance to leave the system without errors # # $1 is the active name, the one to be backed up # # returns 0 if active file was restored, 1 if not # also sets thisFileUpdated for backwards compatibility with existing setup scripts # restoreActiveFile () { thisFileUpdated=false local activeFile="$1" local packageList="$activeFile.package" # look for this package's name in .package list # and remove it if found matchFound=false remainingPackages="" if [ -f "$packageList" ]; then previousPackages=($( cat "$packageList" )) for previousPackage in ${previousPackages[@]}; do if [ "$packageName" == "$previousPackage" ]; then matchFound=true else remainingPackages+="$previousPackage " fi done # no .package file - so proceed with removal anyway else matchFound=true fi # if this package not found - nothing to do if ! $matchFound ; then return 1 fi local baseName=$( basename "$activeFile" ) local previousPatchFile="$previousPatchesDir/$baseName.patch" reversePatchError=false restoreOriginal=false # no other packages so restore the original if [ -z "$remainingPackages" ] ; then restoreOriginal=true # other packages have also modified the active file # attempt to reverse patch the active file # if success, move result into the active position # DO NOT restore original else if [ -e "$previousPatchFile" ]; then tempFile="$tempFileDir/$baseName" cp "$activeFile" "$tempFile" if $patch --reverse -o "$activeFile.tmp" "$tempFile" "$previousPatchFile" &> /dev/null ; then mv -f "$activeFile.tmp" "$activeFile" thisFileUpdated=true else logMessage "CRITICAL: $packageName $baseName - reverse patch failed" rm -f "$activeFile.tmp" reversePatchError=true fi else logMessage "CRITICAL: $packageName $baseName - no prevoius patch file" reversePatchError=true fi # if the reverse patch failed, remove mods from ALL packages #### TODO: DRASTIC but can't think of a way around this if $reversePatchError ; then message1="CRITICAL: $package $baseName could not be uninstalled cleanly" message2=" $remainingPackages must be uninstalled and reinstalled" logMessage "$message1" logMessage "$message2" echo "$message1" >> "$scriptDir/patchErrors" echo "$message2" >> "$scriptDir/patchErrors" # report errors to other packages as well for otherPackage in $remainingPackages ; do otherPkgPatchErrors="$packageRoot/$otherPackage/patchErrors" echo "$message1" >> "$otherPkgPatchErrors" echo "$otherPackage must be uninstalled and reinstalled" >> "$otherPkgPatchErrors" # remove previous patch in other package(s) since it no longer applies rm -f "$previousPatchesRoot/$otherPackage/$baseName.patch" done setInstallFailed $EXIT_PATCH_ERROR "patch error details were saved in $packageName/patchErrors" fi fi # always remove previous patch file rm -f "$previousPatchFile" # restore original if no other packages have modified this active file # or if reverse patch failed if $restoreOriginal || $reversePatchError ; then if [ -e "$activeFile.orig" ]; then mv -f "$activeFile.orig" "$activeFile" thisFileUpdated=true elif [ -f "$activeFile.NO_ORIG" ]; then rm -f "$activeFile" rm -f "$activeFile.NO_ORIG" thisFileUpdated=true fi rm -f "$packageList" # there are other packages, remove only this package from list else grep -v "$packageName" "$packageList" | tee "$packageList" > /dev/null fi # remove file from installed file list if [ -f "$installedFilesList" ]; then grep -v "$activeFile" "$installedFilesList" | tee "$installedFilesList" > /dev/null fi if $thisFileUpdated; then updateRestartFlags "$activeFile" return 0 else return 1 fi } # end restoreActiveFile () # checkFileSets validates the file sets used to install package modifications # # If a file set for the current Venus OS version exists, the replacement files in that file set # are usable as is and no further checks are needed. # The COMPLETE flag file indicates that the file set was validated on the computer creating # the file sets and all replacement files (or symlinks to other file sets) exist. # No checks are needed for a COMPLETE file set. # If not, an attempt is made to create a file set for the current Venus OS version # If the new active files for the new version all match another version # the new file set is populated automatically with replacement files from the other version # and may be used with no further action # If not, new file set is marked INCOMPLETE and installation failure information is set # The package can not be installed on this Venus OS version # # Replacement files that have no original specify an "alternate original" that is used # for version comparisons that locate an appropriate replacement checkFileSets () { # no file sets - nothing to check if ! [ -e "$pkgFileSets" ]; then return; fi # no checks needed if all replacement files exist in the selected file set if [ -f "$fileSet/COMPLETE" ]; then return; fi # sort versionList in reverse version order to make searches faster # since newer versions will most likely contain the desired files to create a new file set local rawVersionList=($(ls -d "$pkgFileSets"/v* 2> /dev/null)) local tempList=() local fs local baseName local version local versionNumber for fs in ${rawVersionList[@]} ; do version=$(basename $fs) versionStringToNumber $version tempList+=("$version:$versionNumber") done local versionList=( $(echo ${tempList[@]} | tr ' ' '\n' | sort -t ':' -r -n -k 2 | uniq ) ) # versioned file sets exist but empty file list if [ ! -z "$versionList" ] && [ -z "$fileList" ]; then setInstallFailed $EXIT_FILE_SET_ERROR "empty file list" touch "$fileSet/INCOMPLETE" rm -f "$fileSet/COMPLETE" return # no versioned file sets - nothing to validate - allow install elif [ -z "$versionList" ]; then return; fi # attempt to create a new file set or validate an existing one not marked as COMPLETE rm -f "$fileSet/INCOMPLETE" # attempt to create file set if it doesn't exist if [ ! -d "$fileSet" ]; then logMessage "creating a new file set for $venusVersion" mkdir "$fileSet" fi for file in ${fileList[@]} ; do baseName=$(basename "$file") # version-independent file exists if [ -e "$versionIndependentFileSet/$baseName" ] || [ -e "$pkgFileSets/$baseName" ]; then # no versioned files (only version-independent found) - skip version checks if [ -z $( find "$fileSet"/v* -name $baseName ) ]; then continue # continue with tests if any versioned files exist else logMessage "WARNING $baseName versioned file exists - version-independent file will be ignored" fi fi # skip checks if replacement file already exists # or if there is no replacement file needed if [ -f "$fileSet/$baseName" ] || [ -f "$fileSet/$baseName.USE_ORIGINAL" ]; then rm -f "$fileSet/$baseName.NO_REPLACEMENT" continue fi local activeFile if [ -f "$altOrigFileDir/$baseName.ALT_ORIG" ]; then activeFile=$(cat "$altOrigFileDir/$baseName.ALT_ORIG") elif [ -f "$pkgFileSets/$baseName.ALT_ORIG" ]; then activeFile=$(cat "$pkgFileSets/$baseName.ALT_ORIG") else activeFile=$file fi if ! [ -e $( dirname "$activeFile" ) ]; then logMessage "no parent directory $activeFile - skipping checks" continue fi # package already installed, use .orig file for comparisons if [ -f "$activeFile.orig" ]; then activeFile="$activeFile.orig" fi # can't process if no original (aka active) file exists in the file set if [ ! -f "$activeFile" ]; then logMessage "ERROR $venusVersion $baseName no active file" touch "$fileSet/$baseName.NO_ACTIVE_FILE" touch "$fileSet/INCOMPLETE" continue fi # if an active file exists look for a match in another file set if [ ! -z "$activeFile" ]; then matchFound=false for entry in ${versionList[@]}; do otherVersion=$(echo $entry | awk -F ':' '{print $1}') # skip this version if [ "$venusVersion" = "$otherVersion" ]; then continue fi otherFile="$pkgFileSets/$otherVersion/$baseName" # skip symbolic links and nonexistent originals if [ ! -f "$otherFile.orig" ] || [ -L "$otherFile.orig" ] ; then continue fi # files match if cmp -s "$activeFile" "$otherFile.orig" > /dev/null ; then matchFound=true break fi done if $matchFound ;then rm -f "$fileSet/$baseName.orig" rm -f "$fileSet/$baseName.NO_ORIG" # if other file set contains a replacement file, link to it if [ -f "$otherFile" ]; then rm -f "$fileSet/$baseName" ln -s "../$otherVersion/$baseName" "$fileSet/$baseName" rm -f "$fileSet/$baseName.NO_REPLACEMENT" rm -f "$fileSet/$baseName.USE_ORIGINAL" # if other file set does not contain a replacement, this one will not either # this IS permitted and handled in the updateActiveFile and restoreActiveFile functions elif [ -f "$otherFile.USE_ORIGINAL" ]; then touch "$fileSet/$baseName.USE_ORIGINAL" rm -f "$fileSet/$baseName.NO_REPLACEMENT" fi # no match to a previous verison - can't create file set automatically # but copy original file to aid manual editing else logMessage "ERROR $venusVersion $baseName no replacement file" cp "$activeFile" "$fileSet/$baseName.orig" touch "$fileSet/$baseName.NO_REPLACEMENT" touch "$fileSet/INCOMPLETE" fi fi done if [ -f "$fileSet/INCOMPLETE" ]; then setInstallFailed $EXIT_FILE_SET_ERROR "incomplete file set for $venusVersion" # if we get this far and fs is not marked INCOMPLETE, then the file set does not need to be checked again next pass else touch "$fileSet/COMPLETE" fi } # builds the installedFilesList and installedServices lists from the package's setup script # this is needed for installs with a prior version of SetupHelper that did not # create the installed... lists # # uninstall... functions then use these lists to uninstall buildUninstallListsfromSetupScript () { # prevent this from running a second time if [ "$uninstallListsAlreadyBuilt" == 'yes' ]; then return fi uninstallListsAlreadyBuilt='yes' local param scriptUninstallFilesList=() scriptUninstallServicesList=() while read -r line || [[ -n "$line" ]]; do commandFound=false read -a params <<< $line numberOfParams=${#params} for (( i=0; i < $numberOfParams; i++ ));do case "${params[i]}" in updateActiveFile) # parameter of intrest is the second one if it exists # otherwise, it should be the first one param=${params[i+2]} if [ -z "$param" ] || [[ $param == \#* ]]; then param=$( echo ${params[i+1]} | sed s?'$qmlDir'?$qmlDir? ) fi if ! [ -z "$param" ] && ! [[ $param == \#* ]]; then # remove any quotes around parameters param=$( echo $param | sed -e 's?"?? g' -e "s?'?? g" ) scriptUninstallFilesList+=("$param") fi ;; installService) # parameter of intrest is the first one if it exists # otherwise, is the packageName param=${params[i+1]} if ! [ -z "$param" ] && ! [[ $param == \#* ]]; then scriptUninstallServicesList+=("$param") else # remove any quotes around parameters param=$( echo $param | sed -e 's?"?? g' -e "s?'?? g" ) scriptUninstallServicesList+=($packageName) fi ;; esac done done < "$scriptDir/setup" } # install / uninstall all files / services functions # # the install... functions are called from endScript if the related flags are set # but may also be called in the setup script # if processing needs to be done after the files are installed # # NOTE: uninstall functions are called in endScrpit during an uninstall # there are no flags to control this ! # # NOTE: only services in the package's serice directory are installed # any services in the package's root diretory are not installed installAllFiles () { if [ -z "$fileListAll" ]; then getFileLists "$pkgFileSets" fi if [ ! -z "$fileListAll" ]; then logMessage "installing files" local file for file in ${fileListAll[@]}; do if $installFailed ; then break; fi updateActiveFile "$file" done fi } # uninstall files from # installed files list if present # and from file lists in the package # and from updateActiveFile calls found in the setup script # this insures complete uninstall even for packages installed # with SetupHelper prior to v6.0 uninstallAllFiles () { local restoreFilesList=() # add installdeFilesList if present (might be empty but that's fine) if [ -f "$installedFilesList" ]; then restoreFilesList=( $( cat "$installedFilesList" ) ) fi # add file lists & calls to installActiveFile in setup script restoreFilesList+=( ${fileListAll[@]} ) buildUninstallListsfromSetupScript restoreFilesList+=( ${scriptUninstallFilesList[@]} ) # remove duplicates if (( ${#restoreFilesList[@]} > 1 )); then restoreFilesList=( $(printf "%s\n" "${restoreFilesList[@]}" | sort -u ) ) fi # uninstall the files if [ ! -z "$restoreFilesList" ]; then logMessage "uninstalling files" local file for file in ${restoreFilesList[@]}; do restoreActiveFile $file done fi } installAllServices () { local serviceList local service # get list of services in the package's service directory if [ -d "$servicesDir" ]; then servicesList=( $( cd "$servicesDir"; ls -d * 2> /dev/null ) ) if ! [ -z "$servicesList" ]; then logMessage "installing services" for service in ${servicesList[@]} ; do if $installFailed; then break; fi installService $service done fi fi } # uninstal services found in the services directory # and restoreServices call in the package's setup script uninstallAllServices () { local tempList=() local servicesList=() local service if [ -f "$installedServicesList" ]; then servicesList=( $( cat "$installedServicesList" ) ) fi # add list from the setup script itself buildUninstallListsfromSetupScript servicesList+=( ${scriptUninstallServicesList[@]} ) # remove duplicates if (( ${#servicesList[@]} > 1 )); then servicesList=( $(printf "%s\n" "${servicesList[@]}" | sort -u ) ) fi # uninstall services if [ ! -z "$servicesList" ]; then logMessage "uninstalling services" for service in ${servicesList[@]} ; do if [ -z "$service" ]; then removeService $packageName else removeService $service fi done fi } # restart the GUI V1 service # begining at about v3.20~18, changes were made to accommodate the gui-v2 # and these changes require different handling of a GUI service restart restartGuiV1Service () { # gui is the older service that runs GUI v1 only if [ -e "/service/gui" ]; then logMessage "restarting GUI V1 (/service/gui)" svc -t "/service/gui" # restart GUI if NOT running v2 or can't determine if GUI v1 or v2 is selected elif [ -e "/service/start-gui" ]; then local guiVersion="$(dbus-send --system --print-reply --dest=com.victronenergy.settings /Settings/Gui/RunningVersion com.victronenergy.BusItem.GetValue | grep variant | awk '{print $3}')" if (( $guiVersion != 2 )); then logMessage "restarting GUI V1 (/service/start-gui)" svc -t "/service/start-gui" fi fi } # for backward compatibility (oler setup scripts) restartGuiService () { restartGuiV1Service } restartGuiV2Service () { # restart GUI if NOT running v1 or can't determine if GUI v1 or v2 is selected if [ -e "/service/start-gui" ]; then local guiVersion="$(dbus-send --system --print-reply --dest=com.victronenergy.settings /Settings/Gui/RunningVersion com.victronenergy.BusItem.GetValue | grep variant | awk '{print $3}')" if (( $guiVersion != 1 )); then logMessage "restarting GUI V2 (/service/start-gui)" svc -t "/service/start-gui" fi fi } # determine how the setup script should exit based on $scriptAction and other flags # services may be restarted here also # # endScript accepts these optional parameters which determines if files and/or services are installed # # 'INSTALL_FILES' causes files from the file lists to be installed # 'INSTALL_SERVICES' causes services in the services directory to be installed # all services must be in the package's services directory # 'ADD_DBUS_SETTINGS' will add/update dBus settings from the DbusSettingsList # any or may be included # do NOT include these if related processing is needed in the setup script ! # instead, call installAllFiles and/or installAllServices in line with the other processing # or call updateActiveFile or installService directly # # this function completes package installation # and sets up conditions for reinstallation following a Venus Os firmware update # # may EXIT or REBOOT within the function - DOES NOT RETURN TO CALLER endScript () { if [ $scriptAction == 'INSTALL' ] && ! $installFailed ; then # do installs as indicated from caller while (( $# > 0 )); do case "$1" in 'INSTALL_FILES') installAllFiles ;; 'INSTALL_SERVICES') installAllServices ;; 'ADD_DBUS_SETTINGS') addAllDbusSettings ;; esac shift done # assume that if we get this far, any command line opitons have already been set touch "$setupOptionsDir/optionsSet" # clear flag preventing auto installs in PackageManager rm -f "$setupOptionsDir/DO_NOT_AUTO_INSTALL" # if script needs to run again, installedVersionFile flag file is removed # script should run again at boot time via reinstallMods if $runAgain ; then logMessage "script will run again at startup" rm -f "$installedVersionFile" # otherwise, installation is complete - update installedVersion else cp "$scriptDir/version" "$installedVersionFile" fi # update rc.local to include call to reinstallMods # do only for SetupHelper since other packages are now installed by PackageManager if [ "$packageName" == "SetupHelper" ]; then if [ ! -f "$rcLocal" ]; then logMessage "creating $rcLocal" cp "$scriptDir/rcS.local" "$rcLocal" chmod +x "$rcLocal" elif [ $(grep -c "blind install" "$rcLocal") -gt 0 ]; then logMessage "REPLACING blind install $rcLocal with the standard one" rm -f "$rcLocal" cp "$scriptDir/rcS.local" "$rcLocal" chmod +x "$rcLocal" elif [ $(grep -c "SetupHelper" "$rcLocal") == 0 ]; then logMessage "adding SetupHelper reinstall script to $rcLocal" sed -e '1,2d' "$scriptDir/rcS.local" >> $rcLocal fi fi fi if [ $scriptAction == 'UNINSTALL' ] ; then if $installFailed ; then logMessage "INSTALL failed - attempting UNINSTALL in endScript" # package was actually uninstalled (not an install failure) - set flag preventing auto installs else touch "$setupOptionsDir/DO_NOT_AUTO_INSTALL" fi # ALWAYS attempt to uninstall files and services uninstallAllFiles uninstallAllServices # flag package not installed since package is being removed rm -f "$installedVersionFile" # when uninstalling SetupHelper, remove EMPTY installed...Lists in all packages # to prevent a future install with an older SetupHelper # confusing a future future uninstall with a new SetupHelper if [ "$packageName" == "SetupHelper" ] && [ -e "$installedFilesDir" ]; then for file in $(ls "$installedFilesDir") ; do if ! [ -s "$installedFilesDir/$file" ]; then rm "$installedFilesDir/$file" fi done # remove lines from rcS.local sed -i -e "/# SetupHelper reinstall/,/fi/d" "$rcLocal" fi fi # setup script signals nothing to do - exit without further action without errors if ! $installFailed && ! $uninstallFailed; then if [ $scriptAction == 'EXIT' ]; then exit $EXIT_SUCCESS elif ! [ $scriptAction == 'INSTALL' ] && ! [ $scriptAction == 'UNINSTALL' ]; then setInstallFailed $EXIT_ERROR "unexpected script action $scriptAction - did not install or uninstall" exit $EXIT_ERROR fi fi # check for reboot and service restarts if $rebootNeeded ; then if $userInteraction ; then if yesNoPrompt "Reboot system now (y) or do it manually later (n): " ; then deferReboot=false else logMessage "system must be rebooted to finish installation and activate components" deferReboot=true fi fi fi # restart services if a reboot won't happen below if ! $rebootNeeded || $deferReboot ; then if $restartGeneratorService ; then logMessage "restarting generator service" if [ -e "svc -t /service/dbus-generator" ]; then svc -t /service/dbus-generator fi if [ -e "svc -t /service/dbus-generator-starter" ]; then svc -t /service/dbus-generator-starter fi fi if $restartSystemCalc ; then logMessage "restarting systemcalc service" svc -t /service/dbus-systemcalc-py fi if $restartDigitalinputs ; then logMessage "restarting digital inputs service" svc -t /service/dbus-digitalinputs fi fi #### TODO: add gui v2 # restart GUI if not doing a reboot below if $restartGui && ! $rebootNeeded && ! $deferReboot; then if $userInteraction ; then if yesNoPrompt "Restart the GUI now (y) or issue a do it manually later (n): " ; then restartGuiV1Service deferGuiRestart=false restartGui=false fi else # GUI restart NOT deferred - do it now if ! $deferGuiRestart ; then restartGuiV1Service deferGuiRestart=false restartGui=false fi fi fi local exitCode=$EXIT_SUCCESS if $installFailed ; then if [ $scriptAction == 'UNINSTALL' ]; then if $uninstallFailed ; then logMessage "CRITICAL: install failed with error $installExitReason" logMessage "CRITICAL: uninstall also failed with error $uninstallExitReason - package state unknown" exitCode=$uninstallExitReason else logMessage "install failed - package has been successfully uninstalled" exitCode=$installExitReason fi else logMessage "install failed during prechecks - no changes were made" exitCode=$installExitReason fi elif $uninstallFailed ; then logMessage "CRITICAL: uninstall failed - package state unknown" exitCode=$installExitReason elif $rebootNeeded ; then if $deferReboot ; then logMessage "reboot needed to complete operaiton" exitCode=$EXIT_REBOOT else logMessage "rebooting ..." reboot fi elif $restartGui && $deferGuiRestart ; then echo "GUI must be restarted to activate changes" exitCode=$EXIT_RESTART_GUI # install/uninstall succeeded else logMessage "complete - no errors" exitCode=$EXIT_SUCCESS fi exit $exitCode } # endScript () ######## this code is executed in-line when CommonResources is sourced # check for reinstall parameter # set $scriptAction to control work following the source command # if "force" is also provided on the command line, then the installedVersionFile is not checked # installedVersionFile contains the installed version (if any) # it is compared to the version file in the package directory # if installedVersionFile is missing or contents are different, the installation will proceed # if the two versions match, there is no need to reinstall the package # we assume a reinstall is always run without benefit of a console (runningAtBoot will be true) # so there will be no prompts and all actions will be automatic # # "deferReboot" signals that endScript should not reboot the system, but return EXIT_REBOOT # assuming the caller will evenutally reboot the system # # "deferGuiRestart" is similar for restarting the GUI # # "install" causes the package to be installed silently # "uninstall" causes the package to be uninstalled silently # # command line parameters may appear in any order # # # logToConsole is set to true in the LogHandler script # It is set to false here the 'auto' parameter is passed on the command line # which indicates this script is NOT being run from the command line # cleanup from previous versions - reinstallScriptsList no loner used rm -f "/data/reinstallScriptsList" # initialize version strings and numbers for future checks if [ -f "$installedVersionFile" ]; then installedVersion=$(cat "$installedVersionFile") versionStringToNumber $installedVersion installedVersionNumber=$versionNumber else installedVersion="" installedVersionNumber=0 fi packageVersionFile="$scriptDir/version" if [ -f "$packageVersionFile" ]; then packageVersion=$(cat "$packageVersionFile") versionStringToNumber $packageVersion packageVersionNumber=$versionNumber else packageVersion="" packageVersionNumber=0 fi # collect command line options reinstall=false deferReboot=false deferGuiRestart=false userInteraction=true runFromPm=false while [ $# -gt 0 ]; do case $1 in "reinstall") reinstall=true ;; "deferReboot") deferReboot=true ;; "deferGuiRestart") deferGuiRestart=true ;; "install") scriptAction='INSTALL' ;; "uninstall") scriptAction='UNINSTALL' ;; "auto") logToConsole=false userInteraction=false ;; "runFromPm") runFromPm=true logToConsole=false userInteraction=false deferReboot=true deferGuiRestart=true ;; "check") # if no other actions were set, set it here # but allow other actions to override this one if [ $scriptAction == 'NONE' ]; then scriptAction='CHECK' fi ;; *) esac shift done # do after logToConsole is enabled/disabled abvove logMessage "--- starting setup script $packageVersion action: $scriptAction" if ! $runFromPm && $logToConsole ; then echo echo "messages are NOT written to log file - only to console" echo fi # packages that require options to proceed unattended # must include the optionsRequried flag file in their package directory # if the flag is present and options haven't been previously set, # SD/USB media will be checked for the package options directory # and copy them into position opitonsRequiredFile="$scriptDir/optionsRequired" optionsSet=false if [ -f $opitonsRequiredFile ]; then if [ -f "$setupOptionsDir/optionsSet" ]; then optionsSet=true # options not set - check media for options if doing a blind install elif [ $scriptAction == 'INSTALL' ]; then mediaList=($(ls /media)) for dir in ${mediaList[@]} ; do altSetupDir="/media/$dir/"$(basename $setupOptionsRoot)"/$packageName" if [ -f "$altSetupDir/optionsSet" ]; then cp -r "$altSetupDir" "$setupOptionsRoot" if [ -f "$setupOptionsDir/optionsSet" ]; then logMessage "options retrieved from SD/USB media" optionsSet=true fi break fi done fi # no command line options are needed - ok to reinstall even if # setup was not run from the command line else optionsSet=true fi # called from reinstallMods at boot time if $reinstall ; then runningAtBoot=true # not installed, do it now if (( installedVersionNumber == 0 )); then scriptAction='INSTALL' # check versions and install only if package version is newer than installed version else # trigger install if version numbers differ if (( installedVersionNumber != packageVersionNumber )); then scriptAction='INSTALL' else exit $EXIT_SUCCESS fi fi # not running from reinstallMods else runningAtBoot=false fi if [ ! -d "$setupOptionsDir" ]; then logMessage "creating package options directory $setupOptionsDir" mkdir -p $setupOptionsDir fi # initialze integer version number for venus version # used below and in checkFileSets versionStringToNumber $venusVersion venusVersionNumber=$versionNumber getFileLists "$pkgFileSets" # create temporary directory for temporary install/uninstall files (unique temp directory in volatile storage) # updateActiveFile always checks this location first for a replacement before checking file sets # tempFileDir is removed in the exit trap above but it is in volatile storage so will be removed on boot anyway tempFileDir=$( mktemp -d ) # patch files previously used to patch a file are stored in /etc/venus # so they track the selected root fs and are erased when Venus OS is updated previousPatchesRoot="/etc/venus/previousPatches" previousPatchesDir="$previousPatchesRoot/$packageName" # do install pre-checks - skip if uninstalling if [ $scriptAction != 'UNINSTALL' ]; then # prevent installing Raspberry Pi packages on other platforms if [ -f "$scriptDir/raspberryPiOnly" ]; then if [[ $machine != *"raspberrypi"* ]]; then setInstallFailed $EXIT_INCOMPATIBLE_PLATFORM "$packageName not compatible with $machine" fi fi # check to see if package is compatible with this Venus version if [ -f "$scriptDir/firstCompatibleVersion" ]; then firstCompatibleVersion=$(cat "$scriptDir/firstCompatibleVersion") # no first compatible version specified - use the default else firstCompatibleVersion='v3.10' fi if [ -f "$scriptDir/obsoleteVersion" ]; then obsoleteVersion=$(cat "$scriptDir/obsoleteVersion") versionStringToNumber $obsoleteVersion obsoleteVersionNumber=$versionNumber else obsoleteVersionNumber=9999999999999999 fi versionStringToNumber $firstCompatibleVersion firstCompatibleVersionNumber=$versionNumber if (( $venusVersionNumber < $firstCompatibleVersionNumber )); then setInstallFailed $EXIT_INCOMPATIBLE_VERSION "$venusVersion before first compatible $firstCompatibleVersion" elif (( $venusVersionNumber >= $obsoleteVersionNumber )); then setInstallFailed $EXIT_INCOMPATIBLE_VERSION "$venusVersion after last compatible $obsoleteVersion" # validate firmware version if valid firmware version file exists elif [ -e "$scriptDir/validFirmwareVersions" ]; then incompatibleVersion=false if ! grep -xq "$venusVersion" "$scriptDir/validFirmwareVersions" ; then incompatibleVersion=true fi # user can override this from command line (for testing new versions mainly) if $incompatibleVersion && $userInteraction ; then if yesNoPrompt "$venusVersion not in the valid firmware list proceed any way (y/n)? " ; then incompatibleVersion=false fi fi if $incompatibleVersion ; then setInstallFailed $EXIT_INCOMPATIBLE_VERSION "$venusVersion not in valid firmware list" fi fi # determine if GUI v1 is installed # Note, it may NOT be running or selected to run!!!! if [ ! -d "/opt/victronenergy/gui" ]; then guiV1present=false else guiV1present=true fi # block installs if any GUI files would be modified and GUI v1 is not present # packages can bypass GUI v1 checks and allow installs even if package contains them if [ -f "$scriptDir/GUI_V1_NOT_REQUIRED" ]; then guiV1required=false # files in the GUI v1 directory are considered mandatory elif (( $( cat "$pkgFileSets/fileList"* 2>/dev/null | grep -c '/gui/' ) > 0 )); then guiV1required=true # look also in setup script elif (( $( grep 'updateActiveFile' "$scriptDir/setup" | grep -c '$qmlDir\|/gui/') > 0 )); then guiV1required=true else guiV1required=false fi if ! $guiV1present && $guiV1required ; then setInstallFailed $EXIT_NO_GUI_V1 "$packageName requires GUI v1" fi # attempting an install without the comand line prompting # and needed options have not been set yet - can't continue if ! $installFailed && [ $scriptAction == 'INSTALL' ]; then if ! $optionsSet ; then setInstallFailed $EXIT_OPTIONS_NOT_SET "required options have not been set" fi fi checkFileSets # checkFileSets created a missing file set and set the INCOMPLETE flag if needed # that is all CHECK needed to do so EXIT HERE !!!! if [ $scriptAction == 'CHECK' ]; then if [ -f "$fileSet/INCOMPLETE" ]; then exit $EXIT_FILE_SET_ERROR fi fi checkPackageDependencies # create patched files for files with VisibleItemModel for older Venus OS versions if ! $installFailed ; then versionStringToNumber "v3.00~14" if (( $venusVersionNumber < $versionNumber )); then logMessage "patching VisibleItemModel to VisualItemModel in all .qml replacements" for file in ${fileListVersionIndependent[@]}; do baseName=$( basename "$file" ) if ! [[ "$baseName" == *.qml ]]; then continue; fi sourceFile="$versionIndependentFileSet/$baseName" if ! [ -f "$sourceFile" ]; then continue; fi if (( $(grep -c "VisibleItemModel" "$sourceFile") == 0 )); then continue; fi sed -e 's/VisibleItemModel/VisualItemModel/' "$sourceFile" > "$tempFileDir/$baseName" done fi # create patched files for all qml files for the change to QtQuick 2 versionStringToNumber "v3.60~18" if (( $venusVersionNumber >= $versionNumber )); then logMessage "changing QtQuick 1.1 to QtQuick 2 in all .qml replacements" for file in ${fileListVersionIndependent[@]}; do baseName=$( basename "$file" ) if ! [[ "$baseName" == *.qml ]]; then continue; fi sourceFile="$versionIndependentFileSet/$baseName" if ! [ -f "$sourceFile" ]; then continue; fi if (( $(grep -c "QtQuick 1.1" "$sourceFile") == 0 )); then continue; fi sed -e 's/QtQuick 1.1/QtQuick 2/' "$sourceFile" > "$tempFileDir/$baseName" done fi fi # create the forward and reverse patched files # used during the actual install and to test if the patch/reverse patch will succeed # done here so PackageManager knows if this will be possible before starting the install # # if this and other packages have both modified the active file, # the patch from this package is first removed # by reverse patching the active file with the PREVIOUS patch file # the new patch is then applied # a test reverse patch insures the patch can be removed in the future # the patch file used for this patch is then saved so it can be used # for the reverse patch on next install/uninstall # # if no other packages have modified the active file, # the patch is applied to .orig file if exists # rather than reverse patching the active file # this maintains compatibility with packages installed with older versions of SetupHelper # use local patch executable - BusyBox version has bugs and does not support all options # include options that are used in all calls # RPI 5 builds use a different binary format so choose the version that works if $( /data/SetupHelper/patch -v &> /dev/null ); then patch="/data/SetupHelper/patch --force --silent --reject-file=/dev/null" elif $( /data/SetupHelper/patchBookworm -v &> /dev/null ); then patch="/data/SetupHelper/patchBookworm --force --silent --reject-file=/dev/null" else patch="" echo "patch executable incompatible with this OS version" > "$scriptDir/patchErrors" setInstallFailed $EXIT_PATCH_ERROR "patch executable incompatible with this OS version - can't continue" # pre-checks only - direct exit if [ $scriptAction == 'CHECK' ]; then exit $EXIT_PATCH_ERROR # script will exit since we are still in pre-checks !! else endScript fi fi if ! $installFailed && ! [ -z "$fileListPatched" ]; then patchErrors=() for activeFile in ${fileListPatched[@]}; do baseName=$( basename $activeFile ) tempActiveFile="$tempFileDir/$baseName.tmp" forwardPatched="$tempFileDir/$baseName.patchedForInstall" currentPatchFile="$tempFileDir/$baseName.currentPatch" previousPatchFile="$previousPatchesDir/$baseName.patch" if ! [ -e "$activeFile" ] ; then logMessage "no active file $activeFile - skipping patch" continue fi rm -f "$currentPatchFile" # check for this and other packages in .package list packageList="$activeFile.package" thisPackageInList=false otherPackagesInList=false if [ -f "$packageList" ]; then previousPackages=$( cat "$packageList" ) for previousPackage in ${previousPackages[@]}; do if [ $packageName == $previousPackage ]; then thisPackageInList=true else otherPackagesInList=true fi done fi patchOk=false skipPatch=false if $thisPackageInList; then if ! $otherPackagesInList ; then # only this package modified active file # ignore any previous patch and patch .orig file if [ -e "$activeFile.orig" ]; then cp "$activeFile.orig" "$tempActiveFile" patchOk=true # use active file if no .orig exists else cp "$activeFile" "$tempActiveFile" patchOk=true fi # this and others have modified the active file # attempt to remove the previous patch for this package # then patch the result elif [ -e "$previousPatchFile" ]; then if $patch --reverse -o "$tempActiveFile" "$activeFile" "$previousPatchFile" &> /dev/null ; then patchOk=true # reverse patch failed else patchErrors+=( "$baseName unable to remove previous patch" ) fi else patchErrors+=( "$baseName no previous patch file" ) fi # this package has not previously modified the active file # patch the active file else cp "$activeFile" "$tempActiveFile" patchOk=true fi patchSuccess=false # a suitable source for the patch was located above if $patchOk; then # attempt to patch the active file with any file ending in .patch # the first one that successfully creates a forward AND reverse patch is used # .patchedForInstall provides the patched file for updateActiveFile patchFiles=( $( ls "$patchSourceDir/$baseName"*.patch ) ) for patchFile in ${patchFiles[@]};do if $patch --forward -o "$forwardPatched" "$tempActiveFile" "$patchFile" &> /dev/null ; then # forward patch succeeded - test reverse patch (both must succeed) if $patch --reverse -o /dev/null "$forwardPatched" "$patchFile" &> /dev/null ; then patchSuccess=true break fi fi done if $patchSuccess ; then # save this so forwardPatched (created above) is used for install # when file is installed, currentPatchFile will be copied to previousPatchFile # for a future uninstall or reinstall cp "$patchFile" "$currentPatchFile" else patchErrors+=( "$baseName patch unsuccesful" ) rm -f "$forwardPatched" fi elif ! $skipPatch ; then patchErrors+=( "$baseName no patch source" ) fi rm -f "$tempActiveFile" done # for activeFile # save errors in patchErrors file if ! [ -z "$patchErrors" ] ; then rm -f "$scriptDir/patchErrors" for patchError in "${patchErrors[@]}"; do logMessage "$patchError" echo "$patchError" >> "$scriptDir/patchErrors" done setInstallFailed $EXIT_PATCH_ERROR "patch error details were saved in $packageName/patchErrors" # pre-checks only - direct exit if [ $scriptAction == 'CHECK' ]; then exit $EXIT_PATCH_ERROR # script will exit since we are still in pre-checks !! else endScript fi # no errors - remove the error history else rm -f "$scriptDir/patchErrors" fi fi # if fileListPatched fi # if [ $scriptAction != 'UNINSTALL' ] # go no further if just checking if [ $scriptAction == 'CHECK' ]; then exit $EXIT_SUCCESS fi # in the Victron images, the root FS is read-only and is a minimum size # in order to install modificaitons, the root partition needs to be # remounted read-write and resized to allow mods to be added # updateRootToReadWrite calls remount-rw.sh or resize2fs.sh # then check to make sure there is sufficient space before allowing installs updateRootToReadWrite # patch files previously used to patch a file are stored in /etc/venus # so they track the selected root fs and are erased when Venus OS is updated if ! [ -e "$previousPatchesDir" ]; then mkdir -p "$previousPatchesDir" fi # relocate previous patch files oldPreviousPatchesDir="$setupOptionsDir/previousPatches" if [ -e "$oldPreviousPatchesDir" ]; then logMessage "relocating previous patches" mv "$oldPreviousPatchesDir"/* "$previousPatchesDir" rm -rf "$oldPreviousPatchesDir" fi unset oldPreviousPatchesDir # done with pre checks # prior to this no system mofications have been made # after this system modifications may occur if $installFailed ; then if ! $userInteraction ; then logMessage "ERROR: errors occured during pre-checks - can't continue" # EXIT HERE if errors occured during pre-checks while running unattended !!!! # the LAST install failure reported to setInstallFailed is used as the exit code exit $installExitReason # EXIT HERE !!!! # command line install request failed - reset scriptAction to NONE # so prompts will be shown - eg # to show the log or # to trigger a manual uninstall elif [ $scriptAction == 'INSTALL' ]; then logMessage "ERROR: install failed during pre-checks - select another action" scriptAction='NONE' fi fi # create installed files (and services) directory if [ ! -d "$installedFilesDir" ]; then mkdir "$installedFilesDir" fi installPreChecks=false #### do standard prompting, automatic install/uninstall then exit if [ "$standardPromptAndActions" == 'yes' ]; then # prompt only if action hasn't been set yet (from command line) if [ "$scriptAction" == 'NONE' ]; then standardActionPrompt fi endScript 'INSTALL_FILES' 'INSTALL_SERVICES' 'ADD_DBUS_SETTINGS' fi # otherwise continue with the setup script #### continue executing the setup script which sourced this file ================================================ FILE: HelperResources/DbusSettingsResources ================================================ #!/bin/bash # DbusSettingsResources for SetupHelper # # contains a functions and variables necessary to access dbus Settings parameters # it should be sourced by scripts setting, creating and removing dbus settings # # dbus Settings is not operational during system boot when some setup scripts may # need to make settings changes # These functions check to see if the settings system is operational and defer # the set/create/remove activity so the calling script may continue # dbus Settings funcitons # These functions encapsulate an interface to dbus Settings # NOTE: dbus Settings resources are not always active when it is necessary for # scripts to make changes or create/remove settings # it is up to the caller to insure dbus Settings resources are active before callling # these functions # a dbus exeption error will be logged if settings are not active yet # updateDbusStringSetting # updateDbusIntSetting # updateDbusRealSetting # updates a dbus setting parameter with a new value # # if the setting does not exist, it is created # but max and min values are not set and the default is "", 0 or 0.0 depending on data type # if these are needed use the dbus command directly # this can also be faster if lots of settings must be created at the same time # # other data types may exist and would need their own function # # $1 is the path to the setting starting with /Settings # $2 is the new value # # if the setting does not yet exist, it is created, then updated to the new value updateDbusStringSetting () { # don't do any work if install has already failed if $installFailed; then return fi dbus-send --system --print-reply=literal --dest=com.victronenergy.settings "$1"\ com.victronenergy.BusItem.GetValue &> /dev/null if (( $? != 0 )); then logMessage "creating dbus Setting $1" dbus -y com.victronenergy.settings / AddSettings "%[ {\"path\":\"$1\", \"default\":\"\"} ]" &> /dev/null fi dbus -y com.victronenergy.settings "$1" SetValue -- "$2" &> /dev/null } updateDbusIntSetting () { # don't do any work if install has already failed if $installFailed; then return fi dbus-send --system --print-reply=literal --dest=com.victronenergy.settings "$1"\ com.victronenergy.BusItem.GetValue &> /dev/null if (( $? != 0 )); then logMessage "creating dbus Setting $1" dbus -y com.victronenergy.settings / AddSettings "%[ {\"path\":\"$1\", \"default\":0} ]" &> /dev/null fi dbus -y com.victronenergy.settings "$1" SetValue -- "$2" &> /dev/null } updateDbusRealSetting () { # don't do any work if install has already failed if $installFailed; then return fi dbus-send --system --print-reply=literal --dest=com.victronenergy.settings "$1"\ com.victronenergy.BusItem.GetValue &> /dev/null if (( $? != 0 )); then logMessage "creating dbus Setting $1" dbus -y com.victronenergy.settings / AddSettings "%[ {\"path\":\"$1\", \"default\":0.0} ]" &> /dev/null fi dbus -y com.victronenergy.settings "$1" SetValue -- "$2" &> /dev/null } # addAllDbusSettings adds settings from DbusSettingsList in the package directory # the format of each line is: # {"path":"/Settings/GuiMods/ShortenTankNames", "default":1, "min":0, "max":1} # min and max are optional addAllDbusSettings () { local settings if [ -f "$scriptDir/DbusSettingsList" ]; then logMessage "updating dbus Settings" while read -r line || [[ -n "$line" ]]; do settings+="$line, " done < "$scriptDir/DbusSettingsList" dbus -y com.victronenergy.settings / AddSettings "%[ $settings ]" &> /dev/null fi } # same as above but removes them # typically settings are retained when removing a package so # the developer must make this call specifically in the setup script's UNINSTALL section # if they wish to remove the settings removeAllDbusSettings () { local settings if [ -f "$scriptDir/DbusSettingsList" ]; then logMessage "removing dbus Settings" while read -r line || [[ -n "$line" ]]; do settings+=$( echo $line | awk -F[:,] '{print $2, ","}' ) done < "$scriptDir/DbusSettingsList" dbus -y com.victronenergy.settings / RemoveSettings "%[ $settings ]" fi } # removeDbusSettings removes the setting from dbus Settings # # all parameters are each a quoted path to the setting to be removed # e.g., removeDbusSettings "/Settings/foo" "/Settings/bar" # (including all settings in one dbus call is much faster) removeDbusSettings () { logMessage "removing dbus Settings $@" local settings=$(echo "$@" | sed -e s_^_\"_ -e s_\$_\"_ -e s_\ _'", "'_g) dbus -y com.victronenergy.settings / RemoveSettings "%[ $settings ]" &> /dev/null } # setSetting updates the dbus setting parameter # the setting must already exist or the update will fail # (the setting can not be created without knowing the data type(s)) # # $1 is the new value # $2 is the setting path setSetting () { # don't do any work if install has already failed if $installFailed; then return fi dbus -y com.victronenergy.settings $2 SetValue $1 &> /dev/null } # move a setting from setup options or from previous dbus Setting # $1 is the setup options path # $2 is the old dbus path (has priority over setup option) # $3 is the new dbus path # dbus paths start with /Settings # if specified, the setup option file must include a value # that value has priority over the old dbus parameter # # setup options can either contain a value or be a flag file # for flag files, the file will be empty and the state of the option # depends on the presence of the file (true) or absense of the file (false) # # Note: this function does NOT create or remove any old option or Setting # use other functions or commands to do so moveSetting () { # don't do any work if install has already failed if $installFailed; then return fi local setupOption="$1" local oldDbusPath=$2 local newDbusPath=$3 if [ ! -z "$oldDbusPath" ]; then oldSetting=$(dbus-send --system --print-reply=literal --dest=com.victronenergy.settings\ $oldDbusPath com.victronenergy.BusItem.GetValue 2> /dev/null | awk '{print $3}') elif [ ! -z $setupOption ]; then if [ -f "$setupOption" ]; then oldSetting=$(cat "$setupOption") # flag file - old setting is true (1) if [ -z $oldSetting ]; then oldSetting=1 fi # file did not exist - assume a false value for a flag file else oldSetting=0 fi else oldSetting="" fi if [ ! -z $oldSetting ] && [ ! -z "$newDbusPath" ]; then dbus -y com.victronenergy.settings $newDbusPath SetValue $oldSetting &> /dev/null fi } ================================================ FILE: HelperResources/EssentialResources ================================================ #!/bin/bash # EssentialResources for SetupHelper # contains a variables necessary for all setup helper scripts # # sourced from IncludeHelpers, packageManagerEnd.sh and reinstallMods # get the full, unambiguous path to this script (and therefore to script that sourced it) scriptDir="$( cd "$(dirname $0)" >/dev/null 2>&1 ; /bin/pwd -P )" packageRoot="$( dirname $scriptDir )" packageName=$( basename "$scriptDir" ) shortScriptName=$(basename "$scriptDir")/$(basename "$0") fullScriptName="$scriptDir/$(basename "$0")" if [ -f "/opt/victronenergy/version" ]; then venusVersion="$(cat /opt/victronenergy/version | head -n 1)" else venusVersion="" fi installedVersionPrefix="/etc/venus/installedVersion-" installedVersionFile="$installedVersionPrefix"$packageName installedFilesDir="/etc/venus/installedModifications" installedFilesList="$installedFilesDir/installedFiles"-$packageName installedServicesList="$installedFilesDir/installedServices"-$packageName # set up pointers to package files # based on the actual package for compatibility with older packages pkgFileSets="$scriptDir/FileSets" fileSet="$pkgFileSets/$venusVersion" versionIndependentFileSet="$pkgFileSets/VersionIndependent" # location of patch files patchSourceDir="$pkgFileSets/PatchSource" altOrigFileDir="$pkgFileSets/AlternateOriginals" servicesDir="$scriptDir/services" # LogHandler functions and variables # enable logging to console # scripts can disable logging by setting # logToConsole to false AFTER sourcing EssentialResources logToConsole=true runFromPm=false # write a message to log file and console logMessage () { # send to stdout if running from PackageManager # it will include the message in its log file if $runFromPm ; then echo "$shortScriptName: $*" # to console elif $logToConsole ; then echo "$*" # to PackageManager log file if no other choices # NOTE: PackageManager's logger service will manage the log file size # however if it is not running /data could eventually fill up else echo "$shortScriptName: $*" | tai64n >> $logFile fi } # create log file and directory tree if it does not exist yet logDir="/var/log/PackageManager" logFile="$logDir/current" if ! [ -e "$logDir" ]; then mkdir -p "$logDir" touch "$logFile" logMessage "creating log file and directory" fi # remove old log file if it still exists oldLogFile="/var/log/SetupHelper" if [ -e "$oldLogFile" ]; then rm -rf "$oldLogFile" fi # rc local file that calls reinstallMods # rcS.local avoids conflicts with mods that blindly replace /data/rc.local rcLocal="/data/rcS.local" # defined exit codes - must be consistent between all setup scripts and reinstallMods # and PackageManager.py EXIT_SUCCESS=0 EXIT_REBOOT=123 EXIT_RESTART_GUI=124 EXIT_ERROR=255 # unknown error EXIT_INCOMPATIBLE_VERSION=254 EXIT_INCOMPATIBLE_PLATFORM=253 EXIT_FILE_SET_ERROR=252 EXIT_OPTIONS_NOT_SET=251 EXIT_RUN_AGAIN=250 EXIT_ROOT_FULL=249 EXIT_DATA_FULL=248 EXIT_NO_GUI_V1=247 EXIT_PACKAGE_CONFLICT=246 EXIT_PATCH_ERROR=245 # directory that holds script's options # options were removed from the script directory so they are preserved when the package is reinstalled setupOptionsRoot="/data/setupOptions" setupOptionsDir="$setupOptionsRoot"/$packageName # packages managed by SetupHelper packageListFile="/data/packageList" qmlDir=/opt/victronenergy/gui/qml # setInstallFailed sets flags to prevent further install steps # and insure the package is uninstalled completely # # $1 indicates the reason for the failure and will evenutally be uused # report the failure reason when exiting the script # # any remaining paremeters are passed to logMessage # and also saved in installFailMessage for others to use # the message is also sent to stderr if not running from the command line # this allows PackageManager to report the full reason for failure # # a setup script can be run from the console, or from another script or program (unattended) # when running from the console # setInstallFailed will report errors and return to the caller # # if running unattended and if the script action is INSTALL # during the precheck period (before any system modification) # setInstallFailed will EXIT WITHOUT RETURNING TO THE CALLER !!!!! # after the precheck period, system modifications may have been made so # the scriptAction is changed to UNINSTALL so the modifictions can be reversed # otherwise, setInstallFailed just logs the error # # installFailed is set here so that additional install operations will not be performed installFailed=false installExitReason=$EXIT_ERROR uninstallExitReason=$EXIT_ERROR installFailMessage="" installPreChecks=true installFailCount=0 uninstallFailed=false setInstallFailed () { local reason (( installFailCount += 1 )) if [ ! -z "$1" ]; then reason=$1 # no reson specified - use the generaic error exit code else reason=EXIT_ERROR fi installFailMessage="${@:2}" if [ ! -z "$installFailMessage" ]; then # output error to stderr if run from PackageManager if $runFromPm ; then echo "$installFailMessage" >&2 # otherwise output to log (and/or stdout/console) else logMessage "ERROR: $installFailMessage" fi fi if [ $scriptAction == 'UNINSTALL' ]; then uninstallExitReason=$reason uninstallFailed=true else installExitReason=$reason installFailed=true fi if [ $scriptAction == 'INSTALL' ]; then # after "pre-checks" system mofifications may already have occured # so an uninstall needs to follow the install if ! $installPreChecks ; then scriptAction='UNINSTALL' # during "pre-checks" failures occur before any system mofifications # if no user interaction EXIT NOW - DO NOT RETURN TO THE CALLER !!!!! elif ! $userInteraction ; then exit $installExitReason fi fi } # set global machine type if [ -f /etc/venus/machine ]; then machine=$(cat /etc/venus/machine) else machine="" setInstallFailed $EXIT_INCOMPATIBLE_PLATFORM "can't determine Venus device type" fi # make sure rootfs is mounted R/W & and resized to allow space for replacement files # arbitrary minimum size of 3 MB # this needs to be called before root fs mods are made. # CommonResources calls this but if you source a subset of helper resources # that script needs to find a place to call updateRootToReadWrite updateRootToReadWrite () { if ! $installFailed; then rootMinimumSize=3 availableSpace=$(df -m / | tail -1 | awk '{print $4}') # remount read-write if (( $(mount | grep ' / ' | grep -c 'rw') == 0 )); then # only remount read-write for CCGX if [ "$machine" == "ccgx" ]; then if [ -f /opt/victronenergy/swupdate-scripts/remount-rw.sh ]; then logMessage "remounting root read-write" /opt/victronenergy/swupdate-scripts/remount-rw.sh fi # remount and resize for other platforms elif [ -f /opt/victronenergy/swupdate-scripts/resize2fs.sh ]; then /opt/victronenergy/swupdate-scripts/resize2fs.sh availableSpace=$(df -m / | tail -1 | awk '{print $4}') logMessage "remounting root read-write and resizing - $availableSpace MB now available" fi # check to see if remount was successful if (( $(mount | grep ' / ' | grep -c 'rw') == 0 )); then setInstallFailed $EXIT_ROOT_FULL "ERROR: unable to remount root read-write - can't continue" fi # root already read-write, attempt to resize if space is limited (CCGX can't resize) elif (( $availableSpace < $rootMinimumSize )) && [ "$machine" != "ccgx" ]; then if [ -f /opt/victronenergy/swupdate-scripts/resize2fs.sh ]; then /opt/victronenergy/swupdate-scripts/resize2fs.sh availableSpace=$(df -m / | tail -1 | awk '{print $4}') logMessage "resized root - $availableSpace MB now available" fi fi fi if ! $installFailed; then # make sure the root partition has space for the package if (( $availableSpace < $rootMinimumSize )); then setInstallFailed $EXIT_ROOT_FULL "no room for modified files on root ($availableSpace MB remaining) - can't continue" fi fi } # convert a version string to an integer to make comparisions easier # the Victron format for version numbers is: vX.Y~Z-large-W # the ~Z portion indicates a pre-release version so a version without it is "newer" than a version with it # the -W portion has been abandoned but was like the ~Z for large builds and is IGNORED !!!! # large builds now have the same version number as the "normal" build # # the version string passed to this function allows for quite a bit of flexibility # any alpha characters are permitted prior to the first digit # up to 3 version parts PLUS a prerelease part are permitted # each with up to 4 digits each -- MORE THAN 4 digits is indeterminate # that is: v0.0.0d0 up to v9999.9999.9999b9999 and then v9999.9999.9999 as the highest priority # any non-numeric character can be used to separate main versions # special significance is assigned to single caracter separators between the numeric strings # b or ~ indicates a beta release # a indicates an alpha release # d indicates an development release # these offset the pre-release number so that b/~ has higher numeric value than any a # and a has higher value than d separator # # a blank version or one without at least one number part is considered invalid # alpha and beta seperators require at least two number parts # # returns 0 if conversion succeeeded, 1 if not # the value integer is returned in $versionNumber # a status text string is returned in $versionStringToNumberStatus # and will include the string passed to the function # as well as the converted number if successful and the type of release detected # or an error reason if not # function versionStringToNumber () { local version="$*" local numberParts local versionParts local numberParts local otherParts local other local number=0 local type='release' # split incoming string into # an array of numbers: major, minor, prerelease, etc # and an array of other substrings # the other array is searched for releasy type strings and the related offest added to the version number read -a numberParts <<< $(echo $version | tr -cs '0-9' ' ') numberPartsLength=${#numberParts[@]} if (( $numberPartsLength == 0 )); then versionNumber=0 versionStringToNumberStatus="$version: invalid, missing major version" return 1 fi if (( $numberPartsLength >= 2 )); then read -a otherParts <<< $(echo $version | tr -s '0-9' ' ') for other in ${otherParts[@]}; do case $other in 'b' | '~') type='beta' (( number += 60000 )) break ;; 'a') type='alpha' (( number += 30000 )) break ;; 'd') type='develop' break ;; esac done fi # if release all parts contribute to the main version number # and offset is greater than all prerelease versions if [ "$type" == "release" ] ; then (( number += 90000 )) # if pre-release, last part will be the pre release part # and others part will be part the main version number else (( numberPartsLength-- )) (( number += 10#${numberParts[$numberPartsLength]} )) fi # include core version number (( number += 10#${numberParts[0]} * 10000000000000 )) if (( numberPartsLength >= 2)); then (( number += 10#${numberParts[1]} * 1000000000 )) fi if (( numberPartsLength >= 3)); then (( number += 10#${numberParts[2]} * 100000 )) fi versionNumber=$number versionStringToNumberStatus="$version:$number $type" return 0 } # compares two version strings # # missing verions are treated as 0 # # returns 0 if they are equal # returns 1 if the first is newer than the second # returns -1 if the second is newer than the first function compareVersions () { local versionNumber2 if [ -z $2 ]; then versionNumber2=0 else versionStringToNumber $2 versionNumber2=$versionNumber fi if [ -z $1 ]; then versionNumber=0 else versionStringToNumber $1 fi if (( versionNumber == versionNumber2 ));then return 0 elif (( versionNumber > versionNumber2 ));then return 1 else return -1 fi } ================================================ FILE: HelperResources/IncludeHelpers ================================================ #!/bin/sh # this script sources helper Resources into the setup script # # for backward compatibility, CommonResources in the SetupHelper directory # links to this file, not CommonResources # CommonResources previously sourced the other files # now sourcing all resource files is done from here # # only the helper files located is the SetupHelper directory are used # previous versions chose between this and a helper file set in the package directory # but changes in SetupHelper to use a local copy of patch made this not possible # # this script should be sourced in the setup script before any other activities pkgDir="$( cd "$(dirname $0)" >/dev/null 2>&1 ; /bin/pwd -P )" pkgRoot="$( dirname "$pkgDir")" pkgName=$( basename $pkgDir ) helperResourcesDir="$pkgRoot/SetupHelper/HelperResources" logDir="/var/log/PackageManager" logFile="$logDir/current" if ! [ -e "$helperResourcesDir" ]; then echo "$pkgName: helper files not found - can't continue" | tee -a "/data/log/SetupHelper" exit 1 fi # if we get here, helper files were located - source the files helperFileList=( EssentialResources ServiceResources DbusSettingsResources ) for file in ${helperFileList[@]}; do if [ -f "$helperResourcesDir/$file" ]; then source "$helperResourcesDir/$file" else echo "$pkgName: helper file $file not found - can't continue" | tee -a "$logFile" exit 1 fi done # now transfer control to CommonResoures - it may not return ! if [ -f "$helperResourcesDir/CommonResources" ]; then source "$helperResourcesDir/CommonResources" else echo "$pkgName: helper file CommonResources not found - can't continue" | tee -a "$logFile" exit 1 fi ================================================ FILE: HelperResources/LogHandler ================================================ # dummy file to prevent failure with older package setup scripts # logging is now in EssentialResources ================================================ FILE: HelperResources/ServiceResources ================================================ #!/bin/bash #!/bin/bash # ServiceManager for SetupHelper # contains a functions to install and remove a package's service # # If an active copy of the service already exists, the run and log/run files are updated # ONLY if there are changes, then the service and/or the logger will be restarted. # This leaves other files managed by supervise untouched. # # in Venus OS starting with v2.90~3, /service is mounted as a tmpfs (RAM disk) # /opt/victronenergy/service is copied to /service ONLY at boot time # so new services need to be copied to /opt/victronenergy/service for boot processing # AND to /service so they run immediately # # svc -u /service/ starts a service that is not already running # svc -d /service/ stops a service and will not restart # these are "temporary" and don't survive a system boot # svc -t /service/ sends the service a TERM command # if the service was up at the time, it restarts # if the service was down at the time, it is NOT started # # the /service//down flag file controls the state of a service at boot time: # if the file exists, the service won't start automatically at boot or when created # if the file does not exist, the service will start at boot or immediately when it is created # # if the services manager (svscan) is not up, or the real service directory does not yet exist # some steps will be skipped to avoid errors in calling daemontools functions # # more info here: # https://cr.yp.to/daemontools/svc.html # https://cr.yp.to/daemontools/supervise.html # https://cr.yp.to/daemontools/svstat.html # https://cr.yp.to/daemontools/svok.html # storage for services # /service is the location where services run from # and is mounted as a temp FS # # contents of serviceDir is copied to /service during boot # but at no other time, so when services are added # they need to be copied to both locations serviceDir="/opt/victronenergy/service" # check to see if services manager is running svscanIsUp () { pgrep -lx svscan &> /dev/null if (( $? == 0 )) ; then return 0 else return 1 fi } # check to see if named service is up serviceIsUp () { if ! svscanIsUp ; then return 1 elif [ ! -e "/service/$1" ]; then return 1 elif [ $(svstat "/service/$1" | awk '{print $2}') == "up" ]; then return 0 else return 1 fi } # # removeService cleanly removes the service # # When uninstalling from within PackageManager, # the PackageManager service is only removed from servicesDir and from installedServicesList # the copy in /services is marked so it will not run again after PackageManager exits removeService () { # no service specified if (( $# < 1 )); then return fi local serviceName="$1" local shutdownService=true if [ "$serviceName" == "PackageManager" ] && $runFromPm ; then shutdownService=false fi if [ -e "$serviceDir/$serviceName" ]; then logMessage "removing $serviceName service" # set flag so PackageManager won't restart after a reboot if ! $shutdownService ; then logMessage "$serviceName service will remain running but will not run again" touch "/service/$serviceName/down" # if PackageManager is running, mark it so it will not restart when it exits if serviceIsUp $serviceName ; then svc -o "/service/$serviceName" fi # stop the service if it is currently running elif serviceIsUp $serviceName ; then svc -d "/service/$serviceName" fi if $shutdownService && serviceIsUp "$serviceName/log" ; then svc -d "/service/$serviceName/log" fi # supervise processes may hang around after removing the service so save info and kill them after removal pids="" while read -u 9 line; do read s uid pid ppid vsz rss tty stime time cmd blah <<< "$line" if [ $cmd == 'supervise' ]; then pids+="$pid " elif [ $cmd == 'multilog' ]; then pids+="$ppid " fi done 9<<< $(ps -lw | grep $serviceName) # remove the service directory rm -rf "$serviceDir/$serviceName" # /service is mounted as tmpfs so the service needs to be removed from /service also # when uninstalling SetupHelper from withing PackageManager, do NOT delete the running service ! # (after the next reboot, /service/PackageManager won't becopied from serviceDir) # (logger remains running until the next boot) if $shutdownService ; then rm -rf "/service/$serviceName" # also kill the supervise processes kill $pids fi fi # remove service from installed services list if [ -f "$installedServicesList" ]; then grep -v "$serviceName" "$installedServicesList" | tee "$installedServicesList" > /dev/null fi } # installService adds the service to the /service directory or updates an existing one # # If the service does not yet exist, it is created # If the service already exists, installService will # update the service files then restart the service and the logger # The service usually starts automatically within a few seconds of creation. # installService waits 10 seconds to see if the service starts on its own # if not, it will be started # # The service may contain a "down" flag file. If present, the service won't be started. # This allows the service to be started manually later. # If the down flag is present the service will not start at boot. # # # $1 is the service name -- that is the name of the service in /service # the package name will be used as the service name if not specified on the command line # # $2 is the directory in the script directory to be copied to the service in /service # (this includes the run and control (down) files) # the default is 'service' in the package directory # # for most packages with one service, the defaults are fine # however if a package needs to install more than one service # then the service name and directory must be specified # installService "PackageManager" "servicePM" # installService "SetupHelper" "serviceSH" # servicePM/run would include a call to /data/SetupHelper/PackageManager.py # serviceSH/run would include a call to /data/SetupHelper/SetupHelper.sh installService () { # don't do any work if install has already failed if $installFailed; then return fi local serviceName="" if (( $# >= 1 )); then serviceName=$1 else serviceName=$packageName fi local servicePath="" if (( $# >= 2 )); then servicePath="$scriptDir/$2" elif [ -e "$servicesDir/$serviceName" ]; then servicePath="$servicesDir/$serviceName" elif [ -e "$scriptDir/service" ]; then servicePath="$scriptDir/service" fi # no service to install if [ ! -e "$servicePath" ]; then setInstallFailed $EXIT_ERROR "service $service not found - can't continue" return fi if [ -L "$serviceDir/$serviceName" ]; then logMessage "removing old $serviceName service (was symbolic link)" removeService $serviceName fi # add service to the installed services list (used for uninstallAll) # do this before actually modifying things just in case there's an error # that way the uninstall is assured echo "$serviceName" >> "$installedServicesList" # service not yet installed, COPY service's directory (run files) to the service directory(s) rm -rf "$serviceDir/$serviceName" cp -R "$servicePath" "$serviceDir/$serviceName" local loggerOk=false if [ -e "$serviceDir/$serviceName/log" ] && ! [ -e "$serviceDir/$serviceName/log/down" ]; then loggerOk=true else logMessage "$serviceName logger disabled (down flag) or does not exist" loggerOk=false fi if [ ! -e "/service/$serviceName" ]; then logMessage "installing $serviceName service" cp -R "$servicePath" "/service/$serviceName" # if down flag is NOT set, check every second for service to start automatically # then start it here if it is not running after 10 seconds if [ -f "$serviceDir/$serviceName/down" ]; then logMessage "$serviceName not (re)started - must be started manually (down flag set)" elif ! svscanIsUp ; then logMessage "services manager (svscan) not yet up - $serviceName should start automatically later" else local delayCount=10 local serviceRunning=false local loggerRunning=false while (( $delayCount > 0 )); do if serviceIsUp $serviceName ; then serviceRunning=true fi if serviceIsUp $serviceName/log ; then loggerRunning=true fi if $serviceRunning && ( $loggerRunning || ! $loggerOk ); then break fi # only report wait once if (( delayCount == 10 )); then echo "waiting for $serviceName service to start" fi sleep 1 (( delayCount-- )) done if $loggerOk ; then if $loggerRunning ; then logMessage "$serviceName logger running" else logMessage "starting $serviceName logger" svc -u "/service/$serviceName/log" fi fi if $serviceRunning; then logMessage "$serviceName service is running" else logMessage "starting $serviceName service" svc -u "/service/$serviceName" fi fi # service already installed - only copy changed files, then restart service if it is running else if [ -f "/service/$serviceName/log/run" ]; then cmp -s "$servicePath/log/run" "/service/$serviceName/log/run" > /dev/null if (( $? != 0 )); then logMessage "updating $serviceName logger run file" cp "$servicePath/log/run" "/service/$serviceName/log/run" fi if [ -e "$servicePath/log/down" ]; then svc -d "/service/$serviceName/log" else rm -f "/service/$serviceName/log/down" if serviceIsUp "$serviceName/log" ; then echo "restarting $serviceName logger" svc -t "/service/$serviceName/log" else logMessage "starting $serviceName logger" svc -u "/service/$serviceName/log" fi fi fi if [ -f "/service/$serviceName/run" ]; then cmp -s "$servicePath/run" "/service/$serviceName/run" > /dev/null if (( $? != 0 )); then logMessage "updating $serviceName run file" cp "$servicePath/run" "/service/$serviceName/run" fi if [ -e "$servicePath/down" ]; then svc -d "/service/$serviceName" else rm -f "/service/$serviceName/down" if serviceIsUp $serviceName ; then echo "restarting $serviceName service" svc -t "/service/$serviceName" else logMessage "starting $serviceName service" svc -u "/service/$serviceName" fi fi fi fi } ================================================ FILE: HelperResources/VersionResources ================================================ # dummy file to prevent failure with older package setup scripts # version resources now in EssentialResources ================================================ FILE: PackageDevelopmentGuidelines.md ================================================ # SetupHelper package development **Kevin Windrem kwindrem@icloud.com** **updated for v6.0** This document provides guidance in developing a package suitable for PackageManager management. When this guide is followed, PackageManager should be able to download the package from GitHub or from media inserted in the GX device and install it (bring it into operation under Venus OS). PackageManager is part of the SetupHelper package. It monitors GitHub for package updates, downloads and installs them automatically (if desired). It also provides a mechanism to add and remove packages from those PackageManager will manage. The user interface for PackageManager is a set of menus in the Venus OS GUI located at Menu / Settings / PackageManager. SetupHelper includes a set of utilities intended to handle the bulk of package installation and uninstallation and also insures the package is reinstalled following a firmware update. A Venus OS update will restore any modified files to factory defaults, so any modifications must be reinstalled following the Venus OS update. There are a few basic requirements for a package to be recognized by PackageManager: 1. The package name must conform to unix file naming convention. To be most compatible, limit the package name to Upper and lower case alpha characters or numbers. **No spaces are allowed.** 1. The package must contain a file named `version`. Specifics of the version are described later. 2. The package should contain a file named `setup`. This should be an executable shell script that will install, uninstall or reinstall the package. SetupHelper includes unix bash shell script extensions that must be included in the setup script in order to be properly managed for reinstall after a Venus OS update. More details on the SetupHelper extension later. 3. The package should contain a file named `gitHubInfo` which should contain the Git Hub user name and default branch separated by `:` E.g., `kwindrem:latest` 4. The primary purpose of the setup script is to install modified files to the Venus OS file system. It is important that the package setup script is also written to **uninstall** the package, restoring any modified files to the factory settings. SetupHelper provides a number of utilities for this. 5. In order for PackageManager to download the package from the internet, it must be contained in a GitHub repository. Any GitHub branch, tag or release may be referenced as long as it is valid. The suggested tag is `latest` which should point to the most recent released version. Additional tags matching a version number may be created in order to allow download of specific versions. Tags or branches such as `beta`, etc may also be created. PackageManager can also use branch names for specific downloads. 6. The package script is run automatically only if a reinstall is required. Code that needs to run constantly or once each boot should use a service. 7. A package may modify (replace) files in the Venus OS to enhance or change behavior. These replacement files should be stored in "file sets" one for each Venus OS version so that the package can be installed regardless of the Venus OS version currently running. More about file sets later. 8. A package may optionally restrict its operation to a range of Venus OS versions. 9. A package may also restrict its operation to a specific platform. Currently, the only restriction is for RaspberryPi platforms. 10. The package may include a ReadMe file describing the package and how to install it. 11. The package may contain a change log indicating what changed in each version. 12. The package may modify GUI files (specifically GUI v1 currently). Normally these modifications are essential to the functionality provided. If GUI v1 is not present on the system, the install will fail. This can be overridden with the `GUI_V1_NOT_REQUIRED` flag file in the package directory. 13. Setup scripts should execute as rapidly as possible. Time spent inside a setup script will delay other activities such as installing other packages and Package Manager will appear to be hung as there is no status update during lengthy operations. For this reason: **Any time-consuming activities such as installing files from the internet or compiling code must be offloaded to a secondary process.** 14. Dependencies on other packages should be identified in the packageDependencies file, described later. ## SetupHelper SetupHelper oversees the installation, uninstallation or reinstallation of the package. The package setup script must call ("source") **IncludeHelpers**, a shell script extension before doing any work to install or uninstall the package: ```bash #### following line incorporates helper resources into this script source "/data/SetupHelper/HelperResources/IncludeHelpers" #### end of lines to include helper resources ``` IncludeHelpers sources the other helper resources. These extensions validate the package for the current Venus OS version and platform and builds a file set for the current Venus OS version if necessary. Beginning with SetupHelper v6.0, file and service and dBus Setting installation is further simplified. These operations will be based on various list(s) and contents of the services directory located in the package directory. Some modifications, such as adding lines to /u-boot/config.txt for Raspberry PIs will still require code in the setup script, but in general **installFiles**, **installServices** and **addDbusSettings** functions will handle the bulk of the activities. These can be called from the setup script or more conveniently, triggered when calling **endScript**. Including the following line in the setup script BEFORE sourcing IncludeHelpers triggers a default prompt for install or uninstall then proceeds with the install/uninstall operations: ```bash standardPromptAndActions='yes' ``` In that case, InstallHelpers never returns to the setup script. If the `standardPromptAndActions` is not set, the helper resources will then hand control back to the setup script. A prompt for user input is then necessary followed by any processing and installation that can't be handled by the list-based functions mentioned above. After any special processing has been performed, the script should call the '**endScript**' function (in CommonResources) in order to exit the script with the proper exit codes. These exit codes are necessary for proper behavior of PackageManager and the automated reinstall operations performed at system boot when the Venus OS is updated. Parameters passed to endScript triggers automatic install/uninstall. (described later) For example: ```bash endScript INSTALL_FILES INSTALL_SERVICE ADD_DBUS_SETTINGS ``` **genericSetupScript** located in the SetupHelper directory will install any package that does not require in-line prompting, install or uninstall code. Simply link to this setup script in that package's directory or copy it there. ## Patched files Package installation has historically replaced files to modify content of the active file. Beginning with v8.0, SetupHelper will attempt to patch an active file in the fileListPatched list rather that replacing it. This allows multiple packages to modify the same file. Files to be patched require entries in the PatchSource directory in the FileSets directory. They also require an entry in the fileListPatched list. A patch (...patch) file is used during the patch process, operating on the active file. The original, unmodified active file(...orig) along with a modified version are needed to create the patch file. If these files are present in the PatchSource directory, updatePackage will attempt to create a patch file, or update it if the orig and modified files have been changed making the .patch file obsolete. **diff -u** is used to create the patch file. A ...patchOptions file can override the default -u option. For example if the patchOptions file is **-U 0** the patch file will not have any context lines. If patchOptions includes **MANUAL**, then **updatePackage** will not attempt to create a patch file. In this case the patch file will need to be created manually. E.g., with: ```bash diff -u foo-1.orig foo-1 > foo-1.patch ``` The patch file then needs to be located in the PatchSource directory. On the GX device, helper resources use the .patch file to create a patched file. These are stored in a temp directory created by **mktemp** and can be referenced with $patchedFiles/foo-1.patch. Some modifications may not be suitable for all Venus OS versions. The patch mechanism allows multiple patch files. E.g.: ```bash PageSettings.qm-1.patch, PageSettings.qm-2.patch ``` Any unique string after the - is an acceptable suffix for the set of patch files. During installation, each patch file will be tested and the first one to succeed is used to patch the active file and unpatch it later during uninstall. During installation, CommonResources attempts to patch the current active file, then reverse patch it to insure that original can be recreated during a future uninstall. If these succeed, the install is allowed to continue. During the uninstall process, the active file replaced by the .orig file if no other packages have modified the file. However, if there are other package modifications still present in the active file, it is "reverse patched" to restore the file to it's state before the package was installed. This would remove the modifications for this package while preserving modifications from other packages. There is a slight possibility that this "reverse patch" could fail. If this occurs, the active file is replaced by the .orig file in order to prevent a system crash due to a missing or incorrectly patched active file. **Modifications from other packages are lost, requiring them to be reinstalled.** There are restrictions that package authors need to keep in mind. These restrictions are due to "context" created for the modification. These are lines prior to and after the modification that are the same with and without the modification. **patch** uses these to locate the proper place in the current active file to apply the patch. 1. Modifications from multiple packages must not overlap. Even modifications that occur at the exact same place in the file might not produce a patch file that succeeds with and without the other package's modifications because the context lines won't match. 2. Modifications at the very beginning or end of a file may not produce a usable patch file. Again, because there is insufficient context. It is important that the author identify other packages that modify the same files and test installation and uninstallation of all packages involved, *and* in different install/uninstall orders. > [!NOTE] > The patch utility provided with Venus OS is part of BusyBox > and has limitations that prevent its use by SetupHelper. Therefore, a > more fully functioned version is included with SetupHelper beginning > with v8.0 and is used by HelperResources. ## Setup script command line options CommonResources checks for optional command line parameters before passing control to the body of the setup script. - **reinstall** indicates this is to be a reinstall. Installed version is compared to the version in the package directory and skips reinstall if they match. User prompting is skipped. (These version checks are now redundant since **reinstallMods** which controls package reinstalls following a Venus OS update also check versions before calling the package setup script.) - **install** indicates that the prompts should be skipped and the installation begun with stored options (more about this later). **uninstall** indicates the prompts should be skipped and the package uninstalled. - **auto** silences all console messages. Progress of the setup is logged. Without the **auto** option progress is also written to the console. - **deferReboot** instructs endScript to skip the automatic system reboot and return indicating a future reboot is needed. - **deferGuiRestart** is as above but for automatic GUI restarts. Without any options, helper resources assumes a user has run the script and sets up the environment to prompt for user input to control additional execution. When the setup script regains control following CommonResources, the variable 'scriptAction' will be set to either NONE, INSTALL or UNINSTALL. NONE indicates that the script should prompt for user input to control further execution. These prompts should be skipped if scriptAction is either INSTALL or UNINSTALL. Note that if an install action fails, scriptAction will be set to UNINSTALL, so scriptAction must be tested again after the install section and perform the uninstall operations. This prevents a partial install from disrupting the system operation. Note that this behavior is not automatic and must be written into all package setup scripts. Venus OS is a "dual root fs" system. That is, the operating system and executable parts of the system reside on one of two root partitions. One partition is active and the other inactive. A Venus OS firmware update installs files to the inactive partition. Then when the update has been verified, the active and inactive portions are swapped and the system rebooted to execute the updated code. If the update is unsuccessful, the swap does not occur and the old executable files continue to run. This prevents a partial or corrupted firmware update from bricking the system. A third partition (/data) holds any persistent information (settings, etc). Packages are stored in the data partition so they survive a Venus OS firmware update, however any system files will be overwritten. At boot time, the **reinstallMods** script is called. Starting with SetupHelper 8.0, **reinstallMods** only installs the PackageManager service. It sets a flag: /etc/venus/REINSTALL_PACKAGES then exists. **PackageManager** then tests this flag file and installs the ALL packages, including the remainder of SetupHelper. This was done to avoid so that all of PackageManager's pre-install checks to be made prior to installing a package. Plus some of the packages available now take several minutes to install resulting in a lag to install all packages after a Venus OS update. Now, status is shown in **PackageManagers**'s menus and in it's log file. When packages are reinstalled, the package setup script is called with the **reinstall**, **auto**, **deferReboot** and **deferGuiRestart** options. **PackageManager** will then trigger a system reboot or GUI restart if any package setup scripts have requested those actions. **PackageManager** acts on requests for system reboot or GUI restarts when an install or uninstall is triggered from it's menu. A user choice to defer reboots and GUI restarts is then provided. A package may sometimes need to manually modify certain files because the automatic mechanisms are too general. For example, adding device tree overlays to `/u-boot/config.txt` must be done in a way that it does not disturb what's currently in the file. All the automatic install mechanism can do is replace the existing file. Code for such modifications goes in the `setupAction == 'INSTALL'` section of code and any restoration code would still go in the `setupAction == 'UNINSTALL'` section. Setup scripts written prior to SetupHelper v6.0 will continue to function. even if the `INSTALL_...` options are set when calling endScript. endScript will attempt to repeat install and uninstall operations but no harm will be done (other than taking additional time). ## Automated install Beginning with SetupHelper v6.0, install and uninstall can usually be handled within CommonResources. Prior to SetupHelper v6.0, the setup script needed code to install and uninstall every modified file or service using the utilities described in the next section. Calls to install and uninstall services was also needed. Installs use "file lists" to install modified files. Each file modified is saved in a modified files list which is then used to uninstall the package. Refer to the section on File Sets below for more details on file lists. All services to be installed by the automatic processes must be located in the services directory. There is no services list since the directory provides the necessary information. An installed services list is created to allow for automatic uninstall. `/data/< package name >/services` Services are still directories with run and log/run files as before. Any service directories found in services will be automatically installed. Automatically installed services are added to an installed services list and will be automatically uninstalled using that list. The name of the directory within services will determine the service name. E.g., the following service directory will create the PackageManager service `/data/SetupHelper/services/PackageManager` Prior to SetupHelper v6.0, services were located in the package directory. The service name was determined by the package name, or specified on the installService line. These services will NOT be automatically installed, so their service directories are moved to services and named appropriately. The modified files and services list are located in /etc/venus. `/etc/venus/installedModifications/installedFiles-` `/etc/venus/installedModifications/installedServices-` This location was chosen because it is removed on a Venus OS update. Running the setup script to uninstall a package will therefore do nothing as expected. If the lists were stored in /data, then an uninstall would attempt to uninstall the files and services again. Generally, that isn't a problem but really should not happen. The package is uninstalled by walking through the installed... lists to restore the original files. Generally, those calls in the setup script are no longer necessary. ## SetupHelper utilities SetupHelper provides a set of utilities to simplify the installation of modified files (called a "replacement") into the active root file system. It pulls the correct replacement file from a "file set" for the current Venus OS version, moves the original out of the way and installs the replacement file. On uninstall, the original file is moved back to the active location (file name) leaving the system in an unmodified state. > [!NOTE] > Starting with v6.0, these utilities may be of less use but are still included. **updateActiveFile** is a function that installs a replacement file as described above. Typically, the replacement file has the same name as the original so the simple form of the command can be used. For example, to replace a GUI file: `updateActiveFile "/opt/victronenergy/gui/qml/main.qml"` If however, the replacement file has a different file name, a second form is used. This may be necessary if the setup script has to build or modify the replacement file. `updateActiveFile "$scriptDir/foo.qml" "/opt/victronenergy/gui/qml/main.qml"` **restoreActiveFile** is a function that undoes the above operation and is called during uninstall with the same file names as were used with updateActiveFile during install: `restoreActiveFile "/opt/victronenergy/gui/qml/main.qml"` **backupActiveFile** is a function that creates the .orig file used by restoreActiveFile but does not update the active file. It is preferable to make modifications in the setup script to a temp file then use **updateActiveFile** as described above but this is not always possible. Use only when something modifies the active file in place: `backupActiveFile "/etc/pointercal"` `ts_calibrate # modifies /etc/pointercal directly` **installService** is a function that installs and starts a dameon service. The service will be placed in the / service directory (or /opt/victronenergy/service for v2.80 or later). A folder in the package folder named 'service' must contain the files that will end up in /service under the service name `installService FooService` Will copy the service directory from the package directory into /service/FooService **removeService** is a function that removes the service which is necessary during an uninstall to restore the system to factory. `removeService FooService` **logMessage** is a function that will log anything of interest. Messages are either sent to stdout or to the PackageManager log file: log file: /var/log/PackageManager/current. Logging is encouraged as it helps debug systems in the field (and while developing the package). Without the **auto** option on the call to the setup script, these messages are also output to the console. To conform to Victron guidelines, messages are sent to stdout in all but a few unavoidable situations: running the script in **auto** from the command line (discouraged) **reinstallMods** or **blindInstall** **blindUninstall**. When scripts are run from PackageManager, stdout is collected and forwarded to the log using python's logging.info () method. This way, a the informaiton is persistant for debugging. When scripts are run from the command line, any messages appear on the console but are **NOT logged**. `logMessage "this text will end up in the log"` **endScript** Function to finish up, prompt the user (if not reinstalling) and exit the script. (Details are described below.) *The following functions simplify the task of getting user input.* **standardActionPrompt** displays a menu of actions and asks the user to choose - It sets scriptAction accordingly and returns - It also handles displaying setup and package logs then asks for an action again - It also handles quitting with no action - the function *exits the shell script* without returning in this case - The basic action prompt includes install, reinstall, quit, display logs (2 choices) - A reinstall option is enabled if the optionsSet option exists - When reinstall is enabled, selecting install, returns a scriptAction of NONE indicating additional prompting may be needed to complete the install - At the end of these prompts, the main script should set scriptAction to INSTALL - If reinstall is selected, the script action is set to INSTALL and the main script should then skip additional prompts and allow options set previously to control the install **yesNoPrompt "prompt "** - Asks the user to answer yes or no to the question - Any details regarding the question should be output before calling yesNoPrompt - **yesNoPrompt** sets **$yesResponse** to true if the answer was 0 if yes and 1 for no so that the return code can be checked rather than checking $yesResponse: ```bash if yesNoPrompt "do it (y/n)?" ; then do stuff for yes response else do stuff for now response fi ``` A set of utilities manages dbus Settings: creating, removing, updating. It is sometimes necessary for the setup script to create dbus Settings so GUI has access to them. This is often the case when the package doesn't run its own service. The following functions will create dbus settings if they do not already exist or update their value if they ```bash updateDbusStringSetting "/Settings/StringSetting" "the new string" updateDbusIntSetting "/Settings/IntegerSetting" 5 updateDbusRealSetting "/Settings/FloatingPointSetting" 6.0 ``` **setSetting** is a function that updates the value of an existing dbus Setting. It is faster than calling one of the above update... functions. The new value can be any data type but strings must be quoted. ```bash setSetting "/Settings/foo" "new string" setSetting "/Settings/bar" 18 ``` The following function removes the settings. Limit the number of settings to about 20 to avoid some being missed (not sure why). It is faster to remove multiple settings at the same time than it is to call 'removeDbusSettings' for each one. ```bash removeDbusSettings "/Settings/foo" "/Settings/bar" ``` ## SetupHelper variables SetupHelper manages or tests a set of variables that control script executions: **$scriptAction** provides direction for the setup script and has the following values: - NONE - setup script should prompt the user for the desired action and set scriptAction accordingly - EXIT - the setup script should exit immediately - INSTALL - the setup script should execute code to install the package - UNINSTALL - the setup script should execute the code to restore the Venus files to stock If installation errors occur within functions in CommonResources, scriptAction will be changed to UNINSTALL. The setup script MUST retest scriptAction after all installation code has been executed so the package can be removed, rather than leaving the system with a partially installed package. **$rebootNeeded** - true signifies a reboot is required after the script is run. The setup script should set **rebootNeeded** to true if a reboot is needed following install/uninstall *The following variables contain useful information but should not be changed by the setup script:* - **$scriptDir** - the full path name to the startup script the script\'s code can use this to identify the location of files stored in the package - **$scriptName** - the basename of the setup script ("setup") - **$reinstallScriptsList** - the file containing a list of scripts to be run at boot to reinstall packages after a Venus software update (by default, this is /data/reinstallScriptsList) - **$installedVersionFile** - the name of the installed version file - **$venusVersion** - the version of VenusOS derived from /opt/victronenergy/version - **$fileList** - the version-dependent location for the replacement files - **$fileListVersionIndependent** - the location for files that are independent of Venus OS version - **$fileListPatched** - the location for files that are to be patched prior to install > [!NOTE] > Prior to SetupHelper v6.0 version-independent replacement files were in $pkgFileSets directory. - **$pkgFileSets** - is the location of all file sets - **$fileSet** - is the location of version-dependent files for the current Venus version - **$runningAtBoot** - true if the script was called from reinstallMods (at boot time) signifying this is to be an unattended (automatic) installation CommonResoures sets this variable based on command line options - **$setupOptionsDir** - the location of any files that control installation These options are maintained in a separate directory so reinstalling the package does not remove them so that a reinstall can proceed without prompting again - **$obsoleteVersion** - prevents installation starting with this Venus OS version - **$firstCompatibleVersion** - prevents installation *before* with this Venus OS version ## Package lists It is usually necessary to create a specific replacement file for different Venus OS versions. This allows the package to be installed regardless of the Venus version. These different replacement file versions are contained in a "file set": a directory with the Venus OS version number as it's name. The collection of file sets is stored in the 'FileSets' directory in the package directory. Some files in a package may not be tied to specific Venus OS versions. These are typically additions to the stock files, and when a single file can be used across all Venus OS versions. Prior to SetupHelper v6.0, these *version-independent* files were contained in the FileSets directory. Starting with v6.0, they are located in the **VersionIndependent** file set. These two file lists are kept separate because they are treated differently. The list **fileList** has always been used by helper resources to guide installation of *version-dependent* files Starting with v6.0, three additional lists have been added: **fileListVersionIndependent** lists all *version-independent* files. Those files exist in the **VersionIndependent** file set. **fileListPatched** is a similar list for any replacement files that are created with the unix patch command. Patch replacements are described above. The file lists consist of one line per file with the full path and name of the file on each line. The **DbusSettingsList** contains a list of dBus Settings to be added to the system as part of this package. Settings are traditionally added via a service but in cases where the package does not have a related service, this mechanism allows there creation or update from the setup script. Lines in **DbusSettingsList** are in the format: ```json {"path":"/Settings/Relay/0/Show", "default":1, "min":0, "max":1} ``` "default" defines the default value plus the data type (1 for an integer, 1.0 for float, "something" for a string) "min" and 'max" are optional and set the range of acceptable values. Refer to Victron dbus documentation for more details File list for version-dependent files only: ``` /data/< package name >/FileSets/fileList ``` File list for version-independent files: ``` /data/< package name >/FileSets/fileListVersionIndependent ``` File list for patched files: ``` /data/\< package name \>/FileSets/fileListPatched ``` File list for dBus Settings: ``` /data/< package name >/DbusSettingsList ``` ### Missing active file directories Recently, Victron Energy has changed the name of some directories which contain files requring modificaiton by the package. For example, /opt/victronenergy/bus-generator-starter was renamed /opt/victronenergy/dbus-generator. In order to accommodate these name changes, SetupHelper v8.23 checks for the existance of the enclosing directory for an active file and skips the update if the directory does not exist. This is logged but installation is allowed to continue. [!NOTE] Developers should review such log entries to insure there are no missing active file updates! In such situations **fileList** and **fileListPatched** must include **both** enclosing directories. E.g.,: ```bash /opt/victronenergy/dbus-generator-starter/startstop.py /opt/victronenergy/dbus-generator/startstop.py ``` ## Version file A package must contain a version file. This is the *package* version, not the Venus OS version. The package version is used by PackageManager to decide if an automatic download is needed by comparing the version from GitHub with the version stored on the system. Likewise, the stored version is compared to the installed version to trigger an automatic install. The version file is a text file with a single line of the form: v1.2, v1.2\~3 v1.2a3. Versions that include a \~ or lettered version are treated as pre-release. - 'd' represents a development release - 'a' represents an alpha release - 'b' or '\~' represents a beta release - none of the above represents a released version Version numbers are prioritized: 'a' is "newer" than 'd', etc. "newer" versions will replace older versions when automatically downloading. Exception: if the branch/ tag set in PackageManger is a specific version (e.g., v.4.6) the stored version must match rather than being older than. Installs always occur if the versions do not match. ## Restricted install The package may optionally contain files that place restrictions on which Venus OS versions or platforms the package may be installed on. If any of these tests fail, the install will also fail! If present **obsoleteVersion** identifies the first version that are not compatible with this package. E.g., of obsoleteVersion contains v7.2 and the current Venus OS version if v8.0, then the package can't be installed. If present **firstCompatibleVersion** identifies the first version that IS compatible with this package. E.g., if firstCompatibleVersion contains v8.0 and the current Venus OS version is v7.2, the package can't be installed. If **firstCompatibleVersion** is not present, SetupHelper uses v3.10 as the first compatible version. Note that if both **firstCompatibleVersion** and **obsoleteVersion** are included in the package directory, the obsoleteVersion must be greater than firstCompatibleVersion. If present **validFirmwareVersions** identifies all versions which have been tested as compatible with the package. It is a list of Venus OS versions, one per line. If this file is present, **firstCompatibleVersion** and **obsoleteVersion** are redundant. If the file **raspberryPiOnly** exists in the package directory, the platform (aka 'machine') MUST be raspberrypi2 or raspberrypi4. If not, installation will be blocked. Many packages modify the GUI file system. With the introduction of gui-v2, some systems will not have the GUI v1 files in place or will not be running the original GUI. If GUI v1 files are required for the package, it's installation will fail. In some cases, the GUI modifications are not essential for package functionality, so if the flag file **GUI_V1_NOT_REQUIRED** is included in the package's root directory the package install will not consider missing GUI v1 files an error. GUI v1 files are those found in /opt/victronenergy/gui. If files from that directory appear in the file list and **GUI_V1_NOT_REQUIRED** is not in the package directory, the install will not be permitted. A check is also made in **updateActiveFile** and will force a package uninstall. **NOTE: SetupHelper will allow an install if the GUI v1 files are present on the system. However, GUI v1 may not currently be running in which case, the user will not have access to the added/modified menus.** Failed installs force an UNINSTALL. ## Package conflict management Prior to SetupHelper v6.12, packages may interact with each other in undesirable ways. For example, one package that modifies the same file as another will install over the other package, removing the first package's modifications. Uninstalling either package will result in the stock file being used. SetupHelper v6.12 adds logic to prevent this from happening. If a package attempts to modify the same file as another package, the install will fail and the package will be uninstalled. Beginning with v8.0, multiple packages may be able to modify the same active file. See the section on patching files above. The **packageDependencies** file located in the package directory defines basic requirements that would prevent the package from being installed. Each line of the file includes a package name and whether that package must be installed or uninstalled in order for the package to be installed. For example the file for RemoteGPIO might be: ``` RpiGpioSetup uninstalled GuiMods installed ``` These lines tell SetupHelper to block the install of RemoteGPIO unless RpiGpioSetup is uninstalled and unless GuiMods is installed. Note that no changes to other package installations occurs so it would be acceptable for the dependency file for RpiGpioSetup to also specify that RemoteGPIO should be uninstalled. This way only one can be installed but the user has that choice. This mechanism is simple but has a drawback: **Package authors must manually check for conflicts and coordinate with other package authors on how best to address the conflict.** ## endScript The **endScript** function must be called at the end of the setup script. It determines the return code used by the caller (like PackageManager) to provide the necessary user prompting and to control reboot and service restarts. **endScript** NEVER RETURNS to the caller ! The actions taken by endScript depend on a number of shell variables set previously and on parameters passed when calling the function: - The following parameters are passed from the caller. All optional: - **INSTALL_FILES** causes endScript to install/uninstall based on fileList, fileListVersionIndependent and fileListPatched lists. - **INSTALL_SERVICE** causes endScript to install/uninstall services located in the package services directory. - **ADD_DBUS_SETTINGS** causes endSctipt to perform the file addition or update of dBus Settings based on the DbusSettingsList in the package directory. - If **$runningAtBoot** is true the script will exit with **EXIT_REBOOT** if **$rebootNeeded** is true otherwise, the script will exit with **EXIT_SUCCESS** on success. - If **$runningAtBoot** is false (script was run manually), user interaction controls further action If **$rebootNeeded** is true, the function asks if the user wishes to reboot now. If they respond yes, the system will be rebooted. The user may choose to not reboot now if additional installations need to be done first - If **$rebootNeeded** is false, the function notifies the user of any needed actions - If **$restartGui** is true the gui service will be restarted Starting with SetupHelper v6.0, other services will be restarted by endSctipt if related files are changed with updateActiveFile or restoreActiveFile. Refer to the updateRestartFlags function in CommonResources for details. If the setup script is run from the command line (no command line options), **endScript** will prompt the user for a reboot or GUI restart if one is needed. The user can choose to trigger the action now or wait and do it manually later. However, if the script is running autonomously, the action will be triggered from within endScript, *unless* the script was run with **deferReboot** or **deferGuiRestart** on the command line. In this case, the action is not performed but the script exits with the appropriate exit code. PackageManager and **renstallMods** use the exit code to choose a course of action following all automatic operations. For manual install/uninstall from PackageManger, the user is given the choice to perform the GUI restart or reboot now or do it later. If deferred, a message will be displayed indicating the package isn't fully active ("reboot needed"). For reinstalls following a Venus OS update, **reinstallMods** will reboot the system or restart the GUI after installing SetupHelper. When the setup script completes an install operation, **endScript** writes the package version to a file in / etc/venus. **endScript** deletes this file during an uninstall. The installed version file written to /etc/venus tells PackageManger which version of each package is installed and running. It also tells the reinstall mechanism to skip SetupHelper reinstall. So reinstall only happens if the installed version file is NOT present or the installed version differs from the package version itself. The latter may be the case if a Venus OS firmware update has occurred. Some packages may need to reboot in the middle of the installation process. For example, if an overlay is needed to test for a specific condition, the setup script should install the overly, but skip the remaining setup, then set the **runAgain** shell variable before calling endScript. endScript then removes the installed version file so the next boot will run the package's setup script again. If an install operation fails, it sets the shell variable **installFailed**. **endScript** will then switch from INSTALL to UNINSTALL to insure that all stock files are restored and the system is not left in a partially modified state. **installFailed** is set by most utility functions but should also be set inside any code in the setup script that detects a failure. Also, any code should test **installFailed** before proceeding with file modifications. ### endScript exit codes The following is a list of exit codes returned when endScript exits: - EXIT_SUCCESS=0 no further action needed - EXIT_REBOOT=123 system reboot needed - EXIT_RESTART_GUI=124 GUI restart needed - EXIT_INCOMPATIBLE_VERSION=254 install failed - version not compatible - EXIT_INCOMPATIBLE_PLATFORM=253 install failed - platform not compatible - EXIT_FILE_SET_ERROR=252 install failed - file set problems - EXIT_OPTIONS_NOT_SET=251 install failed - run setup script from command line - EXIT_RUN_AGAIN=250 partial install - run script again after reboot - EXIT_ROOT_FULL=249 install failed - no room on root - EXIT_DATA_FULL=248 install failed - no room on /data - EXIT_NO_GUI_V1=247 install failed - GUI V1 needed - EXIT_PACKAGE_CONFLICT=246 install of this package blocked by another package - EXIT_ERROR=255 install failed - unknown error ## PackageManager Package Manager includes a set of menus on the GX device menu system that allows the user to view package versions, control automatic package updates and manually install, uninstall, add and remove packages. This provides an alternative to the command line interface for package management. A PackageManager is a python program that runs as a service to do the actual work and to interface with the menus. ### Package Manager menu The first line of this menu provides status for Package Manager, indicating what it is currently doing **Automatic GitHub downloads** controls if packages are automatically downloaded from GitHub. This occurs if a newer version is available. - **On** checks GitHub for a package that is newer than what is stored on the system - **Once** checks GitHub for a package, then downloads are turned off - **Off** disables GitHub downloads GitHub versions are refreshed every 10 minutes if auto downloads is turned on. If auto downloads are off GitHub versions are refreshed once when entering the Package Versions menu. A specific package's GitHub version is also refreshed when entering the Package edit menu. If auto downloads are off, GitHub versions expire after 10 minutes. **Auto install packages** controls whether new versions of a package are automatically installed. Some users may wish to have the system automatically download new updates, but install them manually. In this case, automatic GitHub downloads may be turned on and Auto install packages turned off. Auto install packages also influences whether packages transferred from SD/USB media are automatically installed or just transferred to local storage **Active packages** and **Inactive packages** lead to menus described below **action to finish install/uninstall** appears of a system reboot or GUI restart has been deferred (see the Package editor menu) **Backup & restore settings** leads to the menu described later **microSD / USB** indicates if removable media has been detected and allows it to be ejected prior to removal. **Restart or initialize ...** leads to the menu described below ### Active packages menu Displays all active packages, and allows access to editing the package setup Tapping on one of the entries leads to the Package editor menu Version information is displayed for each package: - GitHub shows the version found on GitHub
 - Stored shows the version stored on the GX device
 - Installed shows the version actually installed and running > [!NOTE] > If the GitHub version is not shown, check the GitHub user and branch/tag, or check your internet connection. ### Package editor menu This menu facilitates manual install, uninstall, package removal as well as changing GitHub access information for the package. **GitHub user** is the name of the GitHub user authoring the package. Normally this won't change. **GitHub branch or tag** allows you to specify a branch or specific tag. The default (typically **latest**) references the latest released version of the package. You can change this field to try out a beta version or revert to a specific version. Once the GitHub branch is changed, PackageManager will update the GitHub version. If auto downloads are active the new version will be downloaded automatically. The status line shows progress of pending operations, conflicts, or prompts for further actions. **Previous** and **Next** step through other packages without leaving this menu. The remaining buttons along the bottom of the menu allow for **Download**, **Install**, **Uninstall** or **Remove**. These operations require a confirmation via **Proceed** or **Cancel** in the status line. **Remove** will remove the package from the active package list and return it to the inactive packages list. Packages that are of no interest can be removed to keep the active list cleaner. Package manager does not allow removing packages unless they are uninstalled first. Package manager DOES permit uninstalling SetupHelper, however this will remove the Package Manager itself. Once removed, the Blind Install mechanism will be needed again !! **Show Conflicts** appears in the status line if package conflicts exist. Pressing that shows a list of conflicts and if possible asks if they should be resolved. If they are, **Proceed** and **Cancel** appear. Pressing **Proceed** will trigger the necessary package installs and uninstalls needed to resolve the conflicts If an operation requires a system reboot or GUI restart, a message appears in the status line. **Now** triggers that operation. **Later** hides the notification without acting on it. This can be handy if you are performing multiple operations. The notification will appear when navigating to other packages. ### Inactive packages menu Displays all INACTIVE packages, i.e., default packages not yet activated or manually remove. The first entry is always \"new\" and allows the operator to enter package name, GitHub user and branch/tag from scratch Additional lines (if any) are default packages (from the defaultPackageList file) If a package is already added to the version list, it will not appear in the Add Package list Tapping on one of the entries leads to the Add package menu ### Add package menu Allows the package name, **GitHub user** and **GitHub branch or tag** to be entered or changed and the package added to the active packages list. These are the same as described above under Package editor menu. Prompting for required information is provided on the status line. Pressing **Proceed** initiates the package add. **Cancel** returns to the Inactive Packages menu. The package name must be unique or the add operation will fail with a prompt indicating to choose a different name. ### Backup & Restore settings menu Saves settings to the settingsBackup file on removable SD/USB media or to local media (`/data/settingsBackup`). restores from same. `/data/SetupHelper/settingsList` is a complete list of settings saved to settingsBackup. Categories are: - GuiMods - SetupHelper / PackageManager - ShutdownManager - SOME Victron stock settings in the following sections
 - Alarms - Gwacs - DigitalInputs
 - Generators - Gui - Pump - Relay - System - SystemSetup - Vrmlogger Additionally, backup and restore the following to/from removable media - Any logo files in `/data/themes/overlay` - Setup script options in `/data/setupOptions` All logs stored in `/data/log` are written to logs.zip on removable media as part of a backup operation The parameters must exist to be saved. The parameters will be created and set to the backed up value during a restore. > [!NOTE] > Victron is working on a more comprehensive mechanism but is not > working reliably yet. The Package manager backup and restore is > temporary and will be removed when the Victron functionality is > working ### Package manager restart/initialize menu This menu provides a quick way to reboot the system (**Restart**), restart the GUI (**Restart GUI**) or initialize Package manager. (**Initialize**). The latter can be used to clean up Package manager's persistent storage. Any custom packages added manually or any GitHub user or branch/tag information will be lost. ### USB/SD updates When the GX device is not connected to the internet, a USB flash drive or microSD card provides an install/upgrade path. To use the USB update process 1. Navigate to the GitHub, repo, click on tags and select the appropriate branch or specific version. 2. Choose the .tar.gz download link.
(Do not download from the Code tab or download the .zip file. These won\'t work.) 3. Copy the archive file to a USB memory stick or microSD card.
Do NOT unpack the archive 4. Repeat this for all packages you wish to install. (These can all be placed on the same media along with the SetupHelper blind install `venus-data.tgz` file) 5. Insert the stick in the GX device. 6. If SetupHelper has not yet been installed, follow the Blind Install process from the ReadMe. Once Package Manager is running, it will transfer and unpack the archive files and update the package list with the new packages. If Auto install packages is turned on, the packages will then be installed > [!NOTE] > No version checks are made for packages found on SD/USB media! > Package Manager is quite content to transfer and install an older > version so make sure you have the latest version especially if your GX > device does not have internet access. ### Package manager control via removable media Besides the menus described above, Package manager can be controlled via "flag" files on removable media. These flag files trigger behavior if they are detected. The file contents is not important, only the existence of the file. **SETTINGS_AUTO_RESTORE** An automatic settings restore will be performed when PackageManager if the file is present. > [!CAUTION] > Leaving this removable media in the system will trigger > settings restore with every boot. You must remove the flash drive > after auto restore **AUTO_EJECT** ALL removable media is ejected after the media is scanned AND if after all transferrers were performed. Removable media can be corrupted if removed while the VRM logger is still writing to it so the drive must be ejected to prevent corruption. A manual eject button is included in the PackageManager menu. Unfortunately, the eject mechanism ejects all removable media, not just a specific one. The VRM logger automatically uses the first removable media found so there is no control over it, and the presence of AUTO_EJECT will eject the media for the logger also. **AUTO_INSTALL_PACKAGES** All packages will be installed even if the Auto Install menu option is turned off. This is generally used only for system deployment (see below). **AUTO_UNINSTALL_PACKAGES** As above, but will uninstall all packages found in /data. This is useful if you do not have command line access and end up with a GUI that is unresponsive or just to clean up a system, returning it (almost) to factory defaults. This flag file overrides AUTO_INSTALL_PACKAGES if both are present The system is rebooted after the uninstall all just to be sure there\'s nothing left behind. **AUTO_INSTALL** If the file AUTO_INSTALL is present in a **package directory**, the package will be installed as if the auto install option is set in the PackageManager menu. Version checks are still performed and DO_NOT_INSTALL is honored. **ONE_TIME_INSTALL** If the file ONE_TIME_INSTALL is present in a **package directory**, the package is automatically installed even if automatic installs are disabled and the DO_NOT_INSTALL flag is set ONE_TIME_INSTALL is removed when the install is performed to prevent repeated installs. Packages may be deployed with this flag set to insure it is installed when a new version is transferred from removable media or downloaded from GitHub **INITIALZE_PACKAGE_MANAGER** The PackageManager's persistent storage is rebuilt (see Initialize above) ## updatePackage **updatePackage** is a unix shell script that runs on the development computer, not the GX device. It is included in the SetupHelper package. The comments at the top of the script provide additional details. Windows will not run this script natively. However Windows 10 apparently supports bash: https://www.howtogeek.com/249966/how-to-install-and-use-the-linux-bash-shell-on-windows-10/ Windows developer options must be enabled. In addition, differences in end of line between platforms may need to be manually adjusted (cr-lf for Windows) for the script to run properly. Another option is to run **updatePackage** on a unix virtual machine or a Raspberry PI running it's native OS. In order to identify changed files, the original file for each replacement must be compared against ALL versions of Venus OS. When changes are detected, a new file set version directory needs to be created and a new \...orig file copied from the stock Venus OS file. This work is done on the computer creating the package, not on the GX device. I use the raspberry Pi images from http://updates.victronenergy.com/feeds/venus/ because these contain the compete file system. Alternatively, the file system can be copied from a running GX device ***prior*** to installing any packages. I create a directory on the managing computer called OriginalFiles, then create a directory for each Venus OS version: v2.81, v2.90\~12, etc. Next, I copy the /etc, /opt and /var/www/venus/styling directories from the Venus OS image to the OriginalFiles/vX.Y\~Z directory. I limit StockFiles to these directories to minimize storage space on the system used to generate the package files. 99% of the files likely to be modified by a package are located there. This is an artificial limit and other parts of the file system may be included if needed. In order to run the necessary checks, Venus OS versions need to be available to the managing computer. Starting with SetupHelper v5.0, it is recommended that a file set exist for all supported Venus OS versions. This speeds installation time because it is not necessary to build a file set for a missing Venus OS version. CommonResources will attempt to create a missing file set, however it may not be possible to create one if version-dependent original files don't match a file for other file sets. **updatePackage** runs through all StockFiles version directories and all package directories and creates file sets in each package. Before running this script, you need to edit the FileSets/fileList\* files to include the files your package will modify. Use full path names to avoid issues. File sets created with **updatePackage** will contain ALL replacement files (or symbolic links to an identical replacement in another file set) in every file set. This change prevents a problem where a matching original file could not be found even though a file set does exist for the current Venus OS version. This resulted in an "incomplete file set" error and failure to install the package. To clear the error, it was necessary to reinstall the package and/or the Venus OS firmware. Prior to v5.0, these symbolic links were missing, and in fact entire file sets might be missing if they can be created from other Venus OS versions. It was then necessary for **\_checkFileSets** to fill in the missing files for the current Venus OS version. Starting with SetupHelper v6.0, **updatePackage** will relocate all version-independent files included in the fileListVersionIndependent list. Prior to v6.0, these files were located in the FileSets directory. Starting with v6.0, these files are located in the VersionIndependent file set **updatePackage** provides an option to relocate alternate original files (described below) to their new locations: AlternateOriginals. If AlternateOriginals is already present, the move will be automatic. **checkFileSets** in CommonResources checks for a COMPLETE flag file before attempting to fill in a missing file set. This saves time since a search is not needed for all replacement files. (In previous versions of SetupHelper, a test was made for all replacement files. This should succeed but takes time.) File sets created with **updatePackage** from an older version of SetupHelper only contain replacement files when the original file changes between versions. Older file sets will also not contain the COMPLETE flag file. Also, the package may not contain a file set for the current Venus OS version. **checkFileSets** will attempt to fill in missing files or build a missing file set by comparing the original files in other file sets with the active file installed by Venus OS. If a match is found, this means the replacement file from the other file set also applies to the current Venus OS version. After running this script, you may find file sets populated with `...NO_REPLACEMENT` files. These indicate where you need to create replacement files for your packages. You will need to add your changes to each replacement file in each file set. Naming: - the replacement file has the extension of the actual file, e.g., PageMain.qml - the original file adds a .orig extension, e.g, PageMain.qml.orig - if no original exists, an empty file with the `.NO_ORIG` file will be created. e.g., `PageMain.qml.NO_ORIG` Existence of a `.NO_ORG` file after running updateFileSets indicates a significant problem. What this says is that the replacement file has no equivalent in a stock system. If the replacement file is the same for all Venus OS versions, simply remove it from fileList and place the replacement in the FileSets directory, not in a version directory. However, if the replacement file differs between Venus OS versions, an alternate original file needs to be used as a reference. For example, if you are creating a new file `PageMainEnhanced.qml`, then you can probably use `PageMain.qml` as the alternate original. Create a file in FileSets named `PageMainEnhanced.qml.ALT_ORIG` with a single line with the full path to the alternate original: `/opt/ victronenergy/qui/qml/PageMain.qml` Sometimes, a replacement file is needed in SOME versions of Venus OS but in others. An empty file in the file set will instruct SetupHelper to use the orig, e.g., `PageMain.qml.USE_ORIGINAL` **Starting with SetupHelper v6.0**, alternate original files can optionally be stored in the alternateOriginals directory in the FileSets directory. This removes clutter from the FileSets directory but the functionality is the same. ## Blind Install By far, the easiest way to install SetupHelper is the \"blind install" which requires no command-line interaction. 1. Download [venus-data.tgz](https://github.com/kwindrem/SetupHelper/raw/main/venus-data.tgz) from the SetupHelper GitHub repo. > **Note:** Mac OS and Safari are set by default to unzip packages. The Open > "safe" files after downloading (bottom of Safari Preferences > General) must be disabled in order to retain the zip file. 2. copy it to the root of a freshly formatted SD card or USB memory stick 3. place the media in the GX device (Cerbo, CCGX, etc) 4. reboot the GX device and allow the system to display the GUI > If you are running Venus OS v2.90 and beyond you should find the > Package Manager menu at the bottom of the Settings menu 5. you should remove the media at this point. Mechanisms are in place to prevent reinstallation, but removal is still a good idea If you are running Venus OS **prior to v2.90**, perform these *additional steps*: 6. Reboot the GX device a second time 7. WHILE the GX device is booting, **REMOVE THE MEDIA** from the GX device *to prevent the next reboot from starting the process all over again*. Failure to do so could disable reinstalls following a Venus OS firmware update !!! You should find the Package Manager menu at the bottom of the Settings menu venus-data.tgz is available here: https://github.com/kwindrem/SetupHelper/raw/main/venus-data.tgz > [!CAUTION] > Prior to v2,90, this mechanism overwrites /data/rcS.local! > > If you are using rcS.local to perform boot-time activities, > /data/rcS.local must be recreated following this \"blind\" install > > Note that SetupHelper also uses /data/rcS.local for reinstallation > following a firmware update so use caution in recreating rcS.local. ## Blind UNINSTALL A blind uninstall mechanism is provided to recover a system with an unresponsive GUI (white screen) or no ssh/terminal access. This will run all package setup scripts to uninstall that package from system files. The archive for this is named `venus-data.UninstallPackages.tar.gz`. 1. Copy `venus-data.UninstallPackages.tar.gz` to a USB memory stick or SD card 2. Rename the copy to `venus-data.tar.gz` 3. Insert the removable media into the GX device 4. Reboot, wait 2 minutes and reboot a second time 5. when the system automatically reboots after the second manual one, remove the media You should eventually see the GUI on the local display if there is one or be able to connect via remote console. > [!CAUTION] > Removing media or power cycling the GX device during the > uninstall, especially if reinstalling firmware could render the system > unresponsive! > > Wait to see the GUI before removing media or power cycling. In addition to uninstalling all packages, the blind uninstall can optionally reinstall VenusOS. To do so, include a `.swu` file for the platform and desired firmware version on the removable media containing the blind uninstall `venus-data.tar.gz` file. Note that a firmware update can take several minutes to complete but will eventually reboot. When the blind uninstall finishes, `venus-data-tar.gz` file on the removable media is renamed to `venus-data.UninstallPackages.tar.gz` so that the blind install will run only once. This renaming is necessary to prevent a loop where the system uninstalls and reboots. ## System automatic configuration and package installation It is possible to use SetupHelper to set up a new system based on a template saved from a working system. 1. Setup the working system the way you want the new system to behave including custom icons, 2. Perform a Settings backup. 3. Remove the flash drive from the GX device and plug into a computer that has internet access. 4. Copy venus-data.tgz from the SetupHelper GitHub repo to the same flash drive. 5. If you wish packages to also be installed, copy the package -latest.tgz file from those repos as well. 6. Create SETTINGS_AUTO_RESTORE on the flash drive (contents don\'t matter - file may be empty). 7. Create AUTO_INSTALL_PACKAGES on the flash drive as well. 8. Place the flash drive into the GX device to be configured and reboot (once for v2.90 or twice for prior versions). 9. REMOVE THE FLASH DRIVE after you have verified that all packages have been installed (check Active packages in PackageManager). ## System recovery It is unlikely, but some users have reported a package install leaving their system unresponsive or with a nonfuncitonal GUI (white screen). In this case, your options depend on the current state of the system. Try the following in this order: - Reboot. This may clear the problem. - If you have a functioning GUI (either locally or via remote console, see if you can access the PackageManager menu. - If so, you can remove packages one at a time from there. - If you find an offending package, post an issue to the GitHub repo for that package and include: - Platform (Cerbo, CCGX, Raspberry PI, etc) - Venus OS firmware version - Run a Settings backup and post the logs.zip file on the removble media. - Remove SetupHelper last since once you do, you loose the PackageManager menus! - If you have terminal or ssh access, try running the package setup scripts to uninstall packages one at a time. - Try reinstalling Venus OS (firmware): - Boot to the previous Venus OS version (in Stored backup firmware), then perform a fresh Online firmware update to the latest version or use the .swu update via removable media. - If you have GUI access, use the Settings / Firmware / Stored backup firmware menu. - If you don't have GUI access, you can also switch to the backup version from the command line: `/opt/victronenergy/swupdate-scripts/set-version.sh 2` - You can also force a firmware update from the command line if you have ssh or terminal access: - For on-line updates: `/opt/victronenergy/swupdate-scripts/check-swupdate.sh -force -update` - For updates from removable media: `/opt/victronenergy/swupdate-scripts/check-swupdate.sh -force -update -offline` - If PackageManager is still running, it will detect a file named AUTO_UNINSTALL_PACKAGES on removable media. - Create a file of that name (no extension, content unimportant) on a USB memory stick or SD card and insert this into the GX device. - The system should eventually reboot. In most cases, this should occur within 1-2 minutes. - After reboot, the system should come up in the stock configuration with no packages installed. - If the system does not reboot, it is likely PackageManager is no longer running, so try other options. - Remember to remove the media containing the AUTO_UNINSTALL_PACKAGES file to this will be repeated the next time PackageManager runs. - Perform the Blind uninstall procedure above. - If you are running on a Raspberry PI, you can reimage the system SD card. - If you have a Cerbo, you can reimage it using this procedure: > **Note:** this will wipe out all settings and you\'ll need to reconfigure the GX device from scratch. The Victron "restore factory default" procedure can be used to will wipe out all settings. You'll need to reconfigure the GX device from scratch. However, it will NOT replace the operating system and Victron application, nor will it uninstall any packages. You will most likely be locked out of ssh access since log-in information and ssh keys are stored in the /data partition which is completely erased by this procedure. For this reason, I do not recommend using this as part of your attempt to recover a system with no GUI. ================================================ FILE: PackageManager.py ================================================ #!/usr/bin/env python #!/usr/bin/env python # # PackageManager.py # Kevin Windrem # # # This program is responsible for # downloading, installing and unstalling packages # package monitor also checks SD cards and USB sticks for package archives # either automatically or manually via the GUI # providing the user with status on installed packages and any updates via the GUI # # It runs as /service/PackageManager # # Persistent storage for packageManager is stored in dbus Settings: # # com.victronenergy.Settings parameters for each package: # /Settings/PackageManager/n/PackageName can not be edited by the GUI # /Settings/PackageManager/n/GitHubUser can be edited by the GUI # /Settings/PackageManager/n/GitHubBranch can be edited by the GUI # /Settings/PackageManager/Count the number of ACTIVE packages (0 <= n < Count) # /Settings/PackageManager/Edit/... GUI edit package set - all fields editable # # /Settings/PackageManager/GitHubAutoDownload set by the GUI to control automatic updates from GitHub # 0 - no GitHub auto downloads (version checks still occur) # 1 - updates enabled - one download update every 10 seconds, then one download every 10 minutes # 3 - one update pass at the fast rate, then to no updates # changing to one of the fast scans, starts from the first package AUTO_DOWNLOADS_OFF = 0 NORMAL_DOWNLOAD = 1 HOURLY_DOWNLOAD = 2 DAILY_DOWNLOAD = 3 ONE_DOWNLOAD = 99 # /Settings/PackageManager/AutoInstall # 0 - no automatic install # 1 - automatic install after download from GitHub or SD/USB # # Additional (volatile) parameters linking packageManager and the GUI are provided in a separate dbus service: # # com.victronenergy.packageManager parameters # /Package/n/GitHubVersion from GitHub # /Package/n/PackageVersion from /data /version from the package directory # /Package/n/InstalledVersion from /etc/venus/isInstalled- # /Package/n/Incompatible indicates the reason the package not compatible with the system # "" if compatible # any other text if not compatible # /Package/n/FileSetOK indicates if the file set for the current version is usable # based on the INCOMPLETE flag in the file set # the GUI uses this to enable/disable the Install button # /Package/n/PackageConflicts (\n separated) list of reasons for a package conflict (\n separated) # # for both Settings and the the dbus service: # n is a 0-based section used to reference a specific package # # a list of default packages that are not in the main package list # these sets are used by the GUI to display a list of packages to be added to the system # filled in from /data/SetupHelper/defaultPackageList, but eliminating any packages already in /data # the first entry (m = 0) is "new" - for a new package # "new" just displays in the packages to add list in the GUI # all package additions are done through /Settings/PackageManager/Edit/... # /Default/m/PackageName # /Default/m/GitHubUser # /Default/m/GitHubBranch # /DefaultCount the number of default packages # # m is a 0-based section used to referene a specific default paclage # # /GuiEditAction is a text string representing the action # set by the GUI to trigger an action in PackageManager # 'install' - install package from /data to the Venus working directories # 'uninstall' - uninstall package from the working directories # 'download" - download package from GutHub to /data # 'add' - add package to package list (after GUI sets .../Edit/... # 'remove' - remove package from list TBD ????? # 'reboot' - reboot # 'restartGui' - restart the GUI # 'INITIALIZE' - install PackageManager's persistent storage (dbus Settings) # so that the storage will be rebuilt when PackageManager restarts # PackageManager will exit when this command is received # 'RESTART_PM' - restart PackageManager # 'gitHubScan' - trigger GitHub version update # sent when entering the package edit menu or when changing packages within that menu # also used to trigger a Git Hub version refresh of all packages when entering the Active packages menu # # the GUI must wait for PackageManager to signal completion of one operation before initiating another # # set by PackageManager when the task is complete # return codes - set by PackageManager # '' - action completed without errors (idle) # 'ERROR' - error during action - error reported in /GuiEditStatus: # unknown error # not compatible with this version # not compatible with this platform # no options present - must install from command line # GUI choices: OK - closes "dialog" # # the following service parameters control settings backup and restore # /BackupMediaAvailable True if suitable SD/USB media is detected by PackageManager # /BackupSettingsFileExist True if PackageManager detected a settings backup file # /BackupSettingsLocalFileExist True if PackageManager detected a settings backup file in /data # /BackupProgress used to trigger and provide status of an operation # 0 nothing happening - set by PackageManager when operaiton completes # 1 set by the GUI to trigger a backup operation media # 2 set by the GUI to trigger a restore operation media # 3 set by PackageManager to indicate a backup to media is in progress # 4 set by PackageManager to indicate a restore from media is in progress # 21 set by the GUI to trigger a backup operation to /data # 22 set by the GUI to trigger a restore operation from /data # 23 set by PackageManager to indicate a backup is in progress to /data # 24 set by PackageManager to indicate a restore from /data is in progress # # setup script return codes EXIT_SUCCESS = 0 EXIT_REBOOT = 123 EXIT_RESTART_GUI = 124 EXIT_INCOMPATIBLE_VERSION = 254 EXIT_INCOMPATIBLE_PLATFORM = 253 EXIT_FILE_SET_ERROR = 252 EXIT_OPTIONS_NOT_SET = 251 EXIT_RUN_AGAIN = 250 EXIT_ROOT_FULL = 249 EXIT_DATA_FULL = 248 EXIT_NO_GUI_V1 = 247 EXIT_PACKAGE_CONFLICT = 246 EXIT_PATCH_ERROR = 245 EXIT_ERROR = 255 # generic error # # # /GuiEditStatus a text message to report edit status to the GUI # # /PmStatus as above for main Package Manager status # # /MediaUpdateStatus as above for SD/USB media transfers # # /Platform a translated version of the platform (aka machine) # machine Platform # ccgx CCGX # einstein Cerbo GX # cerbosgx Cerbo SGX # bealglebone Venus GX # canvu500 CanVu 500 # nanopi Multi/Easy Solar GX # raspberrypi2 Raspberry Pi 2/3 # raspberrypi4 Raspberry Pi 4 # ekrano Ekrano GX # # /ActionNeeded informs GUI if further action is needed following a manual operation # the operator has the option to defer reboots and GUI restarts (by choosing "Later") # '' no action needed # 'reboot' reboot needed # 'guiRestart' GUI restart needed # # the GUI can respond by setting /GuiEditAction to 'reboot' or 'restartGui' # # /Settings/PackageVersion/Edit/ is a section for the GUI to provide information about the a new package to be added # # /data/SetupHelper/defaultPackageList provides an initial list of packages # It contains a row for each package with the following information: # packageName gitHubUser gitHubBranch # If present, packages listed will be ADDED to the package list in /Settings # existing dbus Settings (GitHubUser and GitHubBranch) will not be changed # # this file is read at program start # # Package information is stored in the /data/ directory # # A version file within that directory identifies the version of that package stored on disk # but not necessarily installed # # When a package is installed, the version in the package directory is written to an "installed version" file: # /etc/venus/installedVersion- # this file does not exist if the package is not installed # /etc/venus is chosen to store the installed version because it does NOT survive a firmware update # this will trigger an automatic reinstall following a firmware update # # InstalledVersion is displayed to the user and used for tests for automatic updates # # GitHubVersion is read from the internet if a connection exists. # Once the GitHub versions have been refreshed for all packages, # the rate of refresh is reduced so that all packages are refreshed every 10 minutes. # So if there are 10 packages, one refresh occurs every 60 seconds # Addition of a package or change in GitHubUser or GitHubBranch will trigger a fast # update of GitHub versions # If the package on GitHub can't be accessed, GitHubVersion will be blank # The GUI's Package editor menu will refresh the GitHub version of the current package # when navagating to a new package. This insures the displayed version isn't out of date # GitHub version information is erased 10 minutes after it was last refreshed. # Entering the Package versions menu wil trigger a fast refresh, again to insure the info is up to date. # # # PackageManager downloads packages from GitHub based on the GitHub version and package (stored) versions: # if the GitHub branch is a specific version, automatic download occurs if the versions differ # otherwise the GitHub version must be newer. # the archive file is unpacked to a directory in /data named # -.tar.gz, then moved to /data/, replacing the original # # PackageManager automatically installs the stored verion if the package (stored) and installed versions differ # # Automatic downloads and installs can be enabled separately. # Downloads checks can occur all the time, run once or be disabled # # Package reinstalls following a firmware update are handled as follows: # During system boot, reinstallMods reinstalls SetupHelper if needed # it then sets /etc/venus/REINSTALL_PACKAGES # PackageManager tests this flag and if set, will reinstall all packages # even if automatic installs are disabled. # # Manual downloads and installs triggered from the GUI ignore version checks completely # # In this context, "install" means replacing the working copy of Venus OS files with the modified ones # or adding new files and system services # # Uninstalling means replacing the original Venus OS files to their working locations # # Removed packages won't be checked for automatic install or download # and do not appear in the active package list in the GUI # # Operations that take signficant time are handled in separate threads, decoupled from the package list # Operaitons are placed on a queue with all the information a processing routine needs # # All operations that scan the package list must do so surrounded by # DbusIf.LOCK () and DbusIf.UNLOCK () # and must not consume significant time: no sleeping or actions taking seconds or minutes !!!! # information extracted from the package list must be used within LOCK / UNLOCK to insure # that data is not changed by another thread. # # PackageManager manages flag files in the package's setup options folder: # DO_NOT_AUTO_INSTALL indicates the package was manually removed and PackageManager should not # automatically install it # DO_NOT_AUTO_ADD indicates the package was manually removed and PackageManager should not # automaticlly add it # FORCE_REMOVE instructs PackageManager to remove the package from active packages list # Used rarely, only case is GuiMods setup forcing GeneratorConnector to be removed # this is done only at boot time. # # these flags are stored in /data/setupOptions/ which is non-volatile # and survives a package download and firmware updates # # PackageManager checks removable media (SD cards and USB sticks) for package upgrades or even as a new package # File names must be in one of the following forms: # -.tar.gz # -install.tar.gz # The portion determines where the package will be stored in /data # and will be used as the package name when the package is added to the package list in Settings # # If all criteria are met, the archive is unpacked and the resulting directory replaces /data/ # if not, the unpacked archive directory is deleted # # # PackageManager scans /data looking for new packages # directory names must not appear to be an archive # (include a GitHub branch or version number) (see rejectList below for specifics) # the directory must contain a valid version # the package must not have been manually removed (DO_NOT_AUTO_ADD flag file set) # the file name must be unique to all existing packages # # A new, verified package will be added to the package list and be ready for # manual and automtic updates, installs, uninstalls # # This mechanism handles archives extracted from SD/USB media # # # Packages may optionally include the gitHubInfo file containg GitHub user and branch # gitHubInfo should have a single line of the form: gitHubUser:gitHubBranch, e.g, kwindrem:latest # gitHubUser and gitHubBranch are set from the file's content when it is added to the package list # if the package is already in the package list, gitHubInfo is ignored # if no GitHub information is contained in the package, # an attempt is made to extract it from the defaultPackages list # failing that, the user must add it manually via the GUI # without the GitHub info, no automatic or manual downloads are possible # # alternate user / branch info can be entered for a package # user info probalby should not change # branch info can be changed however to access specific tags/releases # or other branches (e.g., a beta test branch) # # PackageManager has a mechnism for backing up and restoring settings: # SOME dbus Settings # custom icons # backing up gui, SetupHelper and PackageManager logs # # PackageManager checks for several flag files on removable media: # SETTINGS_AUTO_RESTORE # Triggers automatically restore dbus Settings and custom icons # A previous settings backup operation must have been performed # This creates a settingsBackup fiile and icons folder on the removable media # that is used by settings restore (manual or triggered by this flag # # AUTO_INSTALL_PACKAGES # If present on the media, any packages found on the media will be automatically installed # # AUTO_UNINSTALL_PACKAGES # As above but uninstalls INCLUDING SetupHelper !!!! # Only applies if present on media (not in /data or a package direcory) # # AUTO_INSTALL # If present in a package directory, the package is installed # even if the automatic installs are disabled in the PackageManager menu # DO_NOT_AUTO_INSTALL overrides this flag # # ONE_TIME_INSTALL # If present in a package directory, the package is automatically installed # even if automatic installs are diabled and the DO_NOT_AUTO_INSTALL flag is set # This flag file is removed when the install is performed # to prevent repeated installs # Packages may be deployed with this flag set to insure it is installed # when a new version is transferred from removable media or downloaded from GitHub # # AUTO_EJECT # If present, all removable media is ejected after related "automatic" work is finished # # INITIALIZE_PACKAGE_MANAGER # If present, the PackageManager's persistent storage (dbus Settings parameters) are initialized # and PackageManager restarted # On restart, PackageManager will rebuild the dbus Settings from packages found in /data # Only custom Git Hub user and branch information is lost. # # A menu item with the same function as INITIALIZE_PACKAGE_MANAGER is also provided # # classes/instances: # AddRemoveClass # AddRemove runs as a separate thread # # DbusIfClass # DbusIf # # PackageClass # PackageList [] one per package # # UpdateGitHubVersionClass # UpdateGitHubVersion runs as a separate thread # # DownloadGitHubPackagesClass # DownloadGitHub runs as a separate thread # # InstallPackagesClass # InstallPackages runs as a separate thread # # MediaScanClass # MediaScan runs as a separate thread # # global methods: # PushAction () # VersionToNumber () # LocatePackagePath () # AutoRebootCheck () import platform import argparse import logging # constants for logging levels: CRITICAL = 50 ERROR = 40 WARNING = 30 INFO = 20 DEBUG = 10 import sys import signal import subprocess import threading import os import shutil import dbus import time import re import glob import queue from gi.repository import GLib # add the path to our own packages for import sys.path.insert(1, "/data/SetupHelper/velib_python") from vedbus import VeDbusService from settingsdevice import SettingsDevice global DownloadGitHub global InstallPackages global AddRemove global MediaScan global DbusIf global Platform global VenusVersion global VenusVersionNumber global SystemReboot # initialized/used in main, set in mainloop global GuiRestart # initialized in main, set in PushAction and InstallPackage, used in mainloop global WaitForGitHubVersions # initialized in main, set in UpdateGitHubVersion used in mainLoop global InitializePackageManager # initialized/used in main, set in PushAction, MediaScan run, used in mainloop # convert a version string to an integer to make comparisions easier # the Victron format for version numbers is: vX.Y~Z-large-W # the ~Z portion indicates a pre-release version so a version without it is "newer" than a version with it # the -W portion has been abandoned but was like the ~Z for large builds and is IGNORED !!!! # large builds now have the same version number as the "normal" build # # the version string passed to this function allows for quite a bit of flexibility # any alpha characters are permitted prior to the first digit # up to 3 version parts PLUS a prerelease part are permitted # each with up to 4 digits each -- MORE THAN 4 digits is indeterminate # that is: v0.0.0d0 up to v9999.9999.9999b9999 and then v9999.9999.9999 as the highest priority # any non-numeric character can be used to separate main versions # special significance is assigned to single caracter separators between the numeric strings # b or ~ indicates a beta release # a indicates an alpha release # d indicates an development release # these offset the pre-release number so that b/~ has higher numeric value than any a # and a has higher value than d separator # # a blank version or one without at least one number part is considered invalid # alpha and beta seperators require at least two number parts # if only one number part is found the prerelease seperator is IGNORED # # returns the version number or 0 if string does not parse into needed sections def VersionToNumber (version): version = version.replace ("large","L") numberParts = re.split (r'\D+', version) otherParts = re.split (r'\d+', version) # discard blank elements # this can happen if the version string starts with alpha characters (like "v") # of if there are no numeric digits in the version string try: while numberParts [0] == "": numberParts.pop(0) except: pass numberPartsLength = len (numberParts) if numberPartsLength == 0: return 0 versionNumber = 0 releaseType='release' if numberPartsLength >= 2: if 'b' in otherParts or '~' in otherParts: releaseType = 'beta' versionNumber += 60000 elif 'a' in otherParts: releaseType = 'alpha' versionNumber += 30000 elif 'd' in otherParts: releaseType = 'develop' # if release all parts contribute to the main version number # and offset is greater than all prerelease versions if releaseType == 'release': versionNumber += 90000 # if pre-release, last part will be the pre release part # and others part will be part the main version number else: numberPartsLength -= 1 if numberParts [numberPartsLength] != "": versionNumber += int (numberParts [numberPartsLength]) # include core version number if numberPartsLength >= 1 and numberParts [0] != "": versionNumber += int (numberParts [0]) * 10000000000000 if numberPartsLength >= 2 and numberParts [1] != "": versionNumber += int (numberParts [1]) * 1000000000 if numberPartsLength >= 3 and numberParts [2] != "": versionNumber += int (numberParts [2]) * 100000 return versionNumber # get venus version versionFile = "/opt/victronenergy/version" try: file = open (versionFile, 'r') except: VenusVersion = "" VenusVersionNumber = 0 else: VenusVersion = file.readline().strip() VenusVersionNumber = VersionToNumber (VenusVersion) file.close() # PushAction # # some actions are pushed to one of three queues: # InstallPackages.InstallQueue for install, uninstall, check and resolveConflicts actions # Download.Download for download actions # AddRemoveQueue for add and remove actions # GitHubVersion for gitHubScan (GitHub version refresh requiests) # # other actions are handled in line since they just set a global flag # (not pused on any queue) # which is then handled elsewere # # commands are added to the queue from the GUI (dbus service change handler) # and from the main loop (source = 'AUTO') # the queue isolates command triggers from processing because processing # can take seconds or minutes # # command is a string: action:packageName # # action is a text string: Install, Uninstall, Download, Add, Remove, etc # packageName is the name of the package to receive the action # for some actions this may be the null string # # the command and source are pushed on the queue as a tuple # # PushAction sets the ...Pending flag to prevent duplicate operations # for a given package # # returns True if command was accepted, False if not def PushAction (command=None, source=None): parts = command.split (":") theQueue = None if len (parts) >= 1: action = parts[0] else: action = "" if len (parts) >= 2: packageName = parts[1] else: packageName = "" if action == 'download': DbusIf.LOCK ("PushAction 1") package = PackageClass.LocatePackage (packageName) if package != None: package.DownloadPending = True theQueue = DownloadGitHub.DownloadQueue queueText = "Download" # clear the install failure because package contents are changing # this allows an auto install again # but will be disableed if that install fails if source == 'GUI': DbusIf.UpdateStatus ( message=action + " pending " + packageName, where='Editor' ) else: theQueue = None queueText = "" errorMessage = "PushAction Download: " + packageName + " not in package list" logging.error (errorMessage) if source == 'GUI': DbusIf.UpdateStatus ( message=errorMessage, where='Editor' ) DbusIf.AcknowledgeGuiEditAction ( 'ERROR', defer=True ) DbusIf.UNLOCK ("PushAction 1") elif action == 'install' or action == 'uninstall' or action == 'check': DbusIf.LOCK ("PushAction 2") package = PackageClass.LocatePackage (packageName) # flag SetupHelper uninstall for later if packageName == "SetupHelper" and action == 'uninstall': global SetupHelperUninstall SetupHelperUninstall = True if package != None: package.InstallPending = True theQueue = InstallPackages.InstallQueue queueText = "Install" if source == 'GUI': DbusIf.UpdateStatus ( message=action + " pending " + packageName, where='Editor' ) else: theQueue = None queueText = "" errorMessage = "PushAction Install: " + packageName + " not in package list" logging.error (errorMessage) if source == 'GUI': DbusIf.UpdateStatus ( message=errorMessage, where='Editor' ) DbusIf.AcknowledgeGuiEditAction ( 'ERROR', defer=True ) DbusIf.UNLOCK ("PushAction 2") elif action == 'resolveConflicts': theQueue = InstallPackages.InstallQueue queueText = "Install" if source == 'GUI': # note this message will be overwritten by the install and uninstall actions # triggered by this action DbusIf.UpdateStatus ( "resolving conflicts for " + packageName, where='Editor' ) elif action == 'add' or action == 'remove': theQueue = AddRemove.AddRemoveQueue queueText = "AddRemove" if source == 'GUI': DbusIf.UpdateStatus ( message=action + " pending " + packageName, where='Editor' ) elif action == 'gitHubScan': theQueue = UpdateGitHubVersion.GitHubVersionQueue queueText = "GitHubVersion" # the remaining actions are handled here (not pushed on a queue) elif action == 'reboot': global SystemReboot SystemReboot = True logging.info ( "received Reboot request from " + source) if source == 'GUI': DbusIf.UpdateStatus ( message=action + " pending " + packageName, where='Editor' ) # set the flag - reboot is done in main_loop return True elif action == 'restartGui': # set the flag - reboot is done in main_loop global GuiRestart GuiRestart = True logging.info ( "received GUI restart request from " + source) if source == 'GUI': DbusIf.UpdateStatus ( "GUI restart pending " + packageName, where='Editor' ) return True elif action == 'INITIALIZE_PM': # set the flag - Initialize will quit the main loop, then work is done in main global InitializePackageManager InitializePackageManager = True logging.info ( "received PackageManager INITIALIZE request from " + source) if source == 'GUI': DbusIf.UpdateStatus ( "PackageManager INITIALIZE pending " + packageName, where='Editor' ) return True elif action == 'RESTART_PM': # set the flag - Initialize will quit the main loop, then work is done in main global RestartPackageManager RestartPackageManager = True logging.info ( "received PackageManager RESTART request from " + source) if source == 'GUI': DbusIf.UpdateStatus ( "PackageManager restart pending " + packageName, where='Editor' ) return True else: if source == 'GUI': DbusIf.UpdateStatus ( message="unrecognized command '" + command + "'", where='Editor' ) DbusIf.AcknowledgeGuiEditAction ( 'ERROR', defer=True ) logging.error ("PushAction received unrecognized command from " + source + ": " + command) return False if theQueue != None: try: theQueue.put ( (command, source), block=False ) return True except queue.Full: logging.error ("command " + command + " from " + source + " lost - " + queueText + " - queue full") return False except: logging.error ("command " + command + " from " + source + " lost - " + queueText + " - other queue error") return False else: return False # end PushAction # LocatePackagePath # # attempt to locate a package directory # # all directories at the current level are checked # to see if they contain a file named 'version' # indicating a package directory has been found # if so, that path is returned # # if a directory NOT containing 'version' is found # this method is called again to look inside that directory # # if nothing is found, the method returns None # # all recursive calls will return with the located package or None # so the original caller will have the path to the package or None def LocatePackagePath (origPath): paths = os.listdir (origPath) for path in paths: newPath = origPath +'/' + path if os.path.isdir(newPath): # found version file, make sure it is "valid" versionFile = newPath + "/version" if os.path.isfile( versionFile ): return newPath else: packageDir = locatePackagePath (newPath) # found a package directory if packageDir != None: return packageDir # nothing found - continue looking in this directory else: continue return None # AddRemoveClass # Instances: # AddRemove (a separate thread) # Methods: # run ( the thread, pulls from AddRemoveQueue) # StopThread () # # some actions called may take seconds or minutes (based on internet speed) !!!! # # the queue entries are: ("action":"packageName") # this decouples the action from the current package list which could be changing # allowing the operation to proceed without locking the list class AddRemoveClass (threading.Thread): def __init__(self): threading.Thread.__init__(self) self.AddRemoveQueue = queue.Queue (maxsize = 50) self.threadRunning = True # AddRemove run (the thread), StopThread # # run is a thread that pulls actions from a queue and processes them # Note: some processing times can be several seconds to a minute or more # due to newtork activity # # run () checks the threadRunning flag and returns if it is False, # essentially taking the thread off-line # the main method should catch the tread with join () # # run () also serves as and idle loop to add packages found in /data (AddStoredPacakges) # this is only called every 3 seconds # and may push add commands onto the AddRemoveQueue # # StopThread () is called to shut down the thread def StopThread (self): self.threadRunning = False self.AddRemoveQueue.put ( ('STOP', ''), block=False ) # AddRemove run () # # process package Add/Remove actions def run (self): global RestartPackageManager changes = False while self.threadRunning: # if package was added or removed, don't wait for queue empty # so package lists can be updated immediately if changes: delay = 0.0 else: delay = 3.0 try: command = self.AddRemoveQueue.get (timeout = delay) except queue.Empty: # adds/removes since last queue empty if changes: DbusIf.UpdateDefaultPackages () # no changes so do idle processing: # add packages in /data that aren't included in package list else: # restart package manager if a duplice name found in PackageList # or if name is not valid DbusIf.LOCK ("AddRemove run") existingPackages = [] duplicateFound = False for (index, package) in enumerate (PackageClass.PackageList): packageName = package.PackageName if packageName in existingPackages or not PackageClass.PackageNameValid (packageName): duplicateFound = True break existingPackages.append (packageName) del existingPackages DbusIf.UNLOCK ("AddRemove run") # exit this thread so no more package adds/removes are possible # PackageManager will eventually reset if duplicateFound: logging.critical ("duplicate " + packageName + " found in package list - restarting PackageManager") RestartPackageManager = True return PackageClass.AddStoredPackages () changes = False continue except: logging.error ("pull from AddRemoveQueue failed") continue if len (command) == 0: logging.error ("pull from AddRemove queue failed - empty comand") continue # thread shutting down if command [0] == 'STOP' or self.threadRunning == False: return # separate command, source tuple # and separate action and packageName if len (command) >= 2: parts = command[0].split (":") if len (parts) >= 2: action = parts[0].strip () packageName = parts[1].strip () else: logging.error ("AddRemoveQueue - no action or no package name - discarding", command) continue source = command[1] else: logging.error ("AddRemoveQueue - no command and/or source - discarding", command) continue if action == 'add': packageDir = "/data/" + packageName if source == 'GUI': user = DbusIf.EditPackage.GitHubUser branch = DbusIf.EditPackage.GitHubBranch else: user = "" branch = "" # try to get GitHub info from package directory if user == "": if os.path.isdir (packageDir): gitHubInfoFile = packageDir + "/gitHubInfo" try: fd = open (gitHubInfoFile, 'r') parts = fd.readline().strip ().split (':') fd.close() except: parts = "" if len (parts) >= 2: user = parts[0] branch = parts[1] # still nothing - try to get GitHub info from default package list if user == "": default = DbusIf.LocateRawDefaultPackage (packageName) if default != None: user = default[1] branch = default[2] if PackageClass.AddPackage (packageName = packageName, source=source, gitHubUser=user, gitHubBranch=branch ): changes = True elif action == 'remove': if PackageClass.RemovePackage ( packageName=packageName ): changes = True else: logging.warning ( "received invalid action " + command + " from " + source + " - discarding" ) # end while True # end run () # end AddRemoveClass # DbusIfClass # Instances: # DbusIf # Methods: # RemoveDbusSettings (class method) # UpdateStatus # various Gets and Sets for dbus parameters # handleGuiEditAction (dbus change handler) # LocateRawDefaultPackage # UpdateDefaultPackages () # ReadDefaultPackagelist () # LOCK () # UNLOCK () # RemoveDbusService () # # Globals: # DbusSettings (for settings that are NOT part of a package) # DbusService (for parameters that are NOT part of a package) # EditPackage - the dbus Settings used by the GUI to hand off information about # a new package # DefaultPackages - list of default packages, each a tuple: # ( packageName, gitHubUser, gitHubBranch) # # DbusIf manages the dbus Settings and packageManager dbus service parameters # that are not associated with any spcific package # # the dbus settings managed here do NOT have a package association # however, the per-package parameters from PackageClass are ADDED to # DbusSettings and dBusService created here !!!! # # DbusIf manages a lock to prevent data access in one thread # while it is being changed in another # the same lock is used to protect data in PackageClass also # this is more global than it needs to be but simplies the locking # # all methods that access must aquire this lock # prior to accessing DbusIf or Package data # then must release the lock # FALURE TO RELEASE THE LOCK WILL HANG OTHER THREADS !!!!! # # default package info is fetched from a file and published to our dbus service # for use by the GUI in adding new packages # the default info is also stored in defaultPackageList [] # LocateRawDefaultPackage is used to retrieve the default from local storage # rather than pulling from dbus or reading the file again to save time class DbusIfClass: # RemoveDbusSettings # remove the dbus Settings paths for package # package Settings are removed # this is called when removing a package # settings to be removed are passed as a list (settingsList) # this gets reformatted for the call to dbus @classmethod def RemoveDbusSettings (cls, settingsList): # format the list of settings to be removed i = 0 while i < len (settingsList): if i == 0: settingsToRemove = '%[ "' + settingsList[i] else: settingsToRemove += '" , "' + settingsList[i] i += 1 settingsToRemove += '" ]' # remove the dbus Settings paths - via the command line try: proc = subprocess.Popen (['dbus', '-y', 'com.victronenergy.settings', '/', 'RemoveSettings', settingsToRemove ], bufsize=-1, stdout=subprocess.PIPE, stderr=subprocess.PIPE) _, stderr = proc.communicate () stderr = stderr.decode ().strip () returnCode = proc.returncode except: logging.error ("dbus RemoveSettings call failed") else: if returnCode != 0: logging.error ("dbus RemoveSettings failed " + str (returnCode)) logging.error ("stderr: " + stderr) # UpdateStatus # # updates the status when the operation completes # the GUI provides three different areas to show status # where specifies which of these are updated # 'PmStatus' # 'Editor' # 'Media' # which determines where status is sent # message is the text displayed # if LogLevel is not 0, message is also written to the PackageManager log # logging levels: (can use numeric value or these variables set at head of module # CRITICAL = 50 # ERROR = 40 # WARNING = 30 # INFO = 20 # DEBUG = 10 # if where = None, no GUI status areas are updated def UpdateStatus ( self, message=None, where=None, logLevel=0 ): if logLevel != 0: logging.log ( logLevel, message ) if where == 'Editor': DbusIf.SetEditStatus ( message ) elif where == 'PmStatus': DbusIf.SetPmStatus ( message ) elif where == 'Media': DbusIf.SetMediaStatus (message) def UpdatePackageCount (self): count = len(PackageClass.PackageList) self.DbusSettings['packageCount'] = count def GetPackageCount (self): return self.DbusSettings['packageCount'] def SetAutoDownloadMode (self, value): self.DbusSettings['autoDownload'] = value def GetAutoDownloadMode (self): return self.DbusSettings['autoDownload'] def GetAutoInstall (self): return self.DbusSettings['autoInstall'] == 1 def SetAutoInstall (self, value): if value == True: dbusValue = 1 else: dbusValue = 0 self.DbusSettings['autoInstall'] = dbusValue def SetPmStatus (self, value): self.DbusService['/PmStatus'] = value def SetMediaStatus (self, value): self.DbusService['/MediaUpdateStatus'] = value def SetDefaultCount (self, value): self.DbusService['/DefaultCount'] = value def GetDefaultCount (self): return self.DbusService['/DefaultCount'] def SetBackupMediaAvailable (self, value): if value == True: dbusValue = 1 else: dbusValue = 0 self.DbusService['/BackupMediaAvailable'] = dbusValue def GetBackupMediaAvailable (self): if self.DbusService['/BackupMediaAvailable'] == 1: return True else: return True def SetBackupSettingsFileExist (self, value): if value == True: dbusValue = 1 else: dbusValue = 0 self.DbusService['/BackupSettingsFileExist'] = dbusValue def SetBackupSettingsLocalFileExist (self, value): if value == True: dbusValue = 1 else: dbusValue = 0 self.DbusService['/BackupSettingsLocalFileExist'] = dbusValue def GetBackupSettingsFileExist (self): if self.DbusService['/BackupSettingsFileExist'] == 1: return True else: return True def SetBackupProgress (self, value): self.DbusService['/BackupProgress'] = value def GetBackupProgress (self): return self.DbusService['/BackupProgress'] # AcknowledgeGuiEditAction # is part of the PackageManager to GUI communication # the GUI set's an action triggering some processing here # via the dbus change handler # PM updates this dbus value when processing completes # signaling either success or failure # # acknowledgements can not be sent from within the GuiEditAction handler # so these are deferred and processed in mainLoop def AcknowledgeGuiEditAction (self, value, defer=False): global DeferredGuiEditAcknowledgement if defer: DeferredGuiEditAcknowledgement = value else: # delay acknowledgement slightly to prevent lockups in the dbus system time.sleep (0.002) self.DbusService['/GuiEditAction'] = value def SetEditStatus (self, message): self.DbusService['/GuiEditStatus'] = message # handleGuiEditAction (internal use only) # # the GUI uses PackageManager service /GuiEditAction # to request the PackageManager perform some action # # this handler disposes of the request quickly by pushing # the command onto a queue or setting a global variable for later processing # errors that occur from the handler thread can not set the GuiEditAction # since the parameter will be set to the value passed to the handler # overriding the one set within the handler thread def handleGuiEditAction (self, path, command): global PushAction # ignore a blank command - this happens when the command is cleared # and should not trigger further action if command == "": pass else: PushAction ( command=command, source='GUI' ) return True # True acknowledges the dbus change - otherwise dbus parameter does not change # search RAW default package list for packageName # and return the pointer if found # otherwise return None # # Note: the raw default package list is built during init # then never changes so LOCK/UNLOCK is NOT needed # # rawDefaultPackages is a list of tuples: # (packageName, gitHubUser, gitHubBranch) # # if a packageName match is found, the tuple is returned # otherwise None is retuned def LocateRawDefaultPackage (self, packageName): for default in self.rawDefaultPackages: if packageName == default[0]: return default return None # UpdateDefaultPackages # # refreshes the defaultPackageList to include only packages NOT be in PackageList # this also updates the dbus default packages used by the GUI Add Package menu def UpdateDefaultPackages (self): DbusIf.LOCK ("UpdateDefaultPackages") # don't touch "new" entry (index 0) index = 1 oldDefaultCount = len (self.defaultPackageList) for default in self.rawDefaultPackages: # if not in the main package list, add it to the default package list name = default[0] if PackageClass.LocatePackage (name) == None: user = default[1] branch = default[2] prefix = '/Default/' + str (index) + '/' # this entry already exists - update it if index < oldDefaultCount: # name has changed, update the entry (local and dbus) if (name != self.defaultPackageList[index][0]): self.defaultPackageList[index] = default self.DbusService[prefix + 'PackageName'] = name self.DbusService[prefix + 'GitHubUser'] = user self.DbusService[prefix + 'GitHubBranch'] = branch # path doesn't yet exist, add it else: self.defaultPackageList.append (default) self.DbusService.add_path (prefix + 'PackageName', name ) self.DbusService.add_path (prefix + 'GitHubUser', user ) self.DbusService.add_path (prefix + 'GitHubBranch', branch ) index += 1 self.DbusService['/DefaultCount'] = index # clear out any remaining path values while index < oldDefaultCount: prefix = '/Default/' + str (index) + '/' self.defaultPackageList[index] = ( "", "", "" ) self.DbusService[prefix + 'PackageName'] = "" self.DbusService[prefix + 'GitHubUser'] = "" self.DbusService[prefix + 'GitHubBranch'] = "" index += 1 DbusIf.UNLOCK ("UpdateDefaultPackages") # ReadDefaultPackagelist # # read in the default packages list file and store info locally for faster access later # this list is only used to populate the defaultPackageList which excludes packages that # are in the main Packagelist def ReadDefaultPackagelist (self): try: listFile = open ("/data/SetupHelper/defaultPackageList", 'r') except: logging.error ("no defaultPackageList " + listFileName) else: for line in listFile: parts = line.split () if len(parts) < 3 or line[0] == "#": continue self.rawDefaultPackages.append ( ( parts[0], parts[1], parts[2] ) ) listFile.close () # LOCK and UNLOCK - capitals used to make it easier to identify in the code # # these protect the package list from changing while the list is being accessed # # locked sections of code should execute quickly to minimize impact on other threads # # failure to UNLOCK will result in a LOCK request from another thread timing out # # lock requests that time out result in PackageManager exiting immediately without cleanup # supervise will then restart it def LOCK (self, name): requestTime = time.time() reportTime = requestTime while True: if self.lock.acquire (blocking=False): # here if lock was acquired return else: time.sleep (0.1) currentTime = time.time() # waiting for 5 seconds - timeout if currentTime - requestTime > 5.0: logging.critical ("timeout waiting for lock " + name + " - restarting PackageManager") os._exit(1) # report waiting every 1 second elif currentTime - reportTime > 0.5: logging.warning ("waiting to aquire lock " + name) reportTime = currentTime def UNLOCK (self, name): try: self.lock.release () except RuntimeError: logging.error ("UNLOCK when not locked - continuing " + name) def __init__(self): self.lock = threading.RLock() settingsList = {'packageCount': [ '/Settings/PackageManager/Count', 0, 0, 0 ], 'autoDownload': [ '/Settings/PackageManager/GitHubAutoDownload', 0, 0, 0 ], 'autoInstall': [ '/Settings/PackageManager/AutoInstall', 0, 0, 0 ], } self.DbusSettings = SettingsDevice(bus=dbus.SystemBus(), supportedSettings=settingsList, timeout = 30, eventCallback=None ) # check firmware version and delay dbus service registration for v3.40~38 and beyond global VenusVersionNumber global VersionToNumber self.DbusService = VeDbusService ('com.victronenergy.packageManager', bus = dbus.SystemBus(), register=False) self.DbusService.add_mandatory_paths ( processname = 'PackageManager', processversion = 1.0, connection = 'none', deviceinstance = 0, productid = 1, productname = 'Package Manager', firmwareversion = 1, hardwareversion = 0, connected = 1) self.DbusService.add_path ( '/MediaUpdateStatus', "", writeable = True ) self.DbusService.add_path ( '/GuiEditStatus', "", writeable = True ) self.DbusService.add_path ( '/GuiEditAction', "", writeable = True, onchangecallback = self.handleGuiEditAction ) # initialize default package list to empty - entries will be added later self.DbusService.add_path ('/DefaultCount', 0 ) # a special package used for editing a package prior to adding it to Package list self.EditPackage = PackageClass ( section = "Edit", packageName = "" ) self.rawDefaultPackages = [] self.defaultPackageList = [] # create first default package, place where a new package is entered from scratch self.defaultPackageList.append ( ("new", "", "") ) self.DbusService.add_path ( "/Default/0/PackageName", "new" ) self.DbusService.add_path ( "/Default/0/GitHubUser", "" ) self.DbusService.add_path ( "/Default/0/GitHubBranch", "" ) # used to notify the GUI that an action is required to complete a manual installation # the operator has the option to defer reboot and GUI restart operations # if they do, this parameter is set and a button appears on the main Package manager menu self.DbusService.add_path ( "/ActionNeeded", '' ) self.DbusService.add_path ( '/BackupMediaAvailable', 0, writeable = True ) self.DbusService.add_path ( '/BackupSettingsFileExist', 0, writeable = True ) self.DbusService.add_path ( '/BackupSettingsLocalFileExist', 0, writeable = True ) self.DbusService.add_path ( '/BackupProgress', 0, writeable = True ) # do these last because the GUI uses them to check if PackageManager is running self.DbusService.add_path ( '/PmStatus', "", writeable = True ) global Platform self.DbusService.add_path ( '/Platform', Platform ) self.DbusService.register () # RemoveDbusService # deletes the dbus service def RemoveDbusService (self): logging.info ("shutting down com.victronenergy.packageManager dbus service") self.DbusService.__del__() # end DbusIf # PackageClass # Instances: # one per package # # Methods: # LocatePackage # GetAutoAddOk (class method) # SetAutoAddOk (class method) # SetAutoInstallOk () # settingChangedHandler () # various Gets and Sets # AddPackagesFromDbus (class method) # PackageNameValid (class method) # AddStoredPackages (class method) # AddPackage (class method) # RemovePackage (class method) # UpdateVersionsAndFlags () # # Globals: # PackageList [] - list instances of all packages # DbusSettings (for per-package settings) # DbusService (for per-package parameters) # DownloadPending # InstallPending # # a package consits of Settings and version parameters in the PackageMonitor dbus service # all Settings and parameters are accessible via set... and get... methods # so that the caller does not need to understand dbus Settings and service syntax # the packageName variable maintains a local copy of the dBus parameter for speed in loops # section passed to init can be either a int or string ('Edit') # an int is converted to a string to form the dbus setting paths # # the dbus settings and service parameters managed here are on a per-package basis class PackageClass: # list of instantiated Packages PackageList = [] # search PackageList for packageName # and return the package pointer if found # otherwise return None # # Note: this method should be called with LOCK () set # and use the returned value before UNLOCK () # to avoid unpredictable results @classmethod def LocatePackage (cls, packageName): for package in PackageClass.PackageList: if packageName == package.PackageName: return package return None # this set of methods manages the flag files that control # automaticly adding and installing packages # if a package is manually removed, it should not # be readded automatically # ditto for manual uninstall # @classmethod def GetAutoAddOk (cls, packageName): if packageName == None: logging.error ("GetAutoAddOk - no packageName") return False flagFile = "/data/setupOptions/" + packageName + "/DO_NOT_AUTO_ADD" if os.path.exists (flagFile): return False else: return True @classmethod def SetAutoAddOk (cls, packageName, state): if packageName == None: logging.error ("SetAutoAddOk - no packageName") return # if package options directory exists set/clear auto add flag # directory may not exist if package was never downloaded or transferred from media # or if package was added manually then never acted on optionsDir = "/data/setupOptions/" + packageName if os.path.exists (optionsDir): flagFile = optionsDir + "/DO_NOT_AUTO_ADD" # permit auto add if state == True: if os.path.exists (flagFile): os.remove (flagFile) # block auto add else: if not os.path.exists (flagFile): # equivalent to unix touch command open (flagFile, 'a').close() def SetAutoInstallOk (self, state): packageName = self.PackageName if packageName == None: logging.error ("SetAutoInstallOk - no packageName") return # if package options directory exists set/clear auto install flag # directory may not exist if package was never downloaded or transferred from media # or if package was added manually then never acted on optionsDir = "/data/setupOptions/" + packageName if os.path.exists (optionsDir): flagFile = optionsDir + "/DO_NOT_AUTO_INSTALL" # permit auto installs if state == True: if os.path.exists (flagFile): os.remove (flagFile) # block auto install else: if not os.path.exists (flagFile): open (flagFile, 'a').close() def SetPackageName (self, newName): self.DbusSettings['packageName'] = newName self.PackageName = newName def SetInstalledVersion (self, version): global VersionToNumber self.InstalledVersion = version self.InstalledVersionNumber = VersionToNumber (version) if self.installedVersionPath != "": DbusIf.DbusService[self.installedVersionPath] = version def SetPackageVersion (self, version): global VersionToNumber self.PackageVersion = version self.PackageVersionNumber = VersionToNumber (version) if self.packageVersionPath != "": DbusIf.DbusService[self.packageVersionPath] = version def SetGitHubVersion (self, version): global VersionToNumber self.GitHubVersion = version self.GitHubVersionNumber = VersionToNumber (version) if self.gitHubVersionPath != "": DbusIf.DbusService[self.gitHubVersionPath] = version def SetGitHubUser (self, user): self.GitHubUser = user self.DbusSettings['gitHubUser'] = user def SetGitHubBranch (self, branch): self.GitHubBranch = branch self.DbusSettings['gitHubBranch'] = branch def SetIncompatible (self, value, details="", resolvable=False): self.Incompatible = value self.IncompatibleDetails = details self.IncompatibleResolvable = resolvable if self.incompatiblePath != "": DbusIf.DbusService[self.incompatiblePath] = value if self.incompatibleDetailsPath != "": DbusIf.DbusService[self.incompatibleDetailsPath] = details if self.IncompatibleResolvablePath != "": if resolvable: DbusIf.DbusService[self.IncompatibleResolvablePath] = 1 else: DbusIf.DbusService[self.IncompatibleResolvablePath] = 0 def settingChangedHandler (self, name, old, new): # when dbus information changes, need to refresh local mirrors if name == 'packageName': self.PackageName = new elif name == 'gitHubBranch': self.GitHubBranch = new if self.PackageName != None and self.PackageName != "": UpdateGitHubVersion.SetPriorityGitHubVersion ( 'package:' + self.PackageName ) elif name == 'gitHubUser': self.GitHubUser = new if self.PackageName != None and self.PackageName != "": UpdateGitHubVersion.SetPriorityGitHubVersion ( 'package:' + self.PackageName ) def __init__( self, section, packageName = None ): # add package parameters if it's a real package (not Edit) if section != 'Edit': section = str (section) self.gitHubVersionPath = '/Package/' + section + '/GitHubVersion' self.packageVersionPath = '/Package/' + section + '/PackageVersion' self.installedVersionPath = '/Package/' + section + '/InstalledVersion' self.incompatiblePath = '/Package/' + section + '/Incompatible' self.incompatibleDetailsPath = '/Package/' + section + '/IncompatibleDetails' self.IncompatibleResolvablePath = '/Package/' + section + '/IncompatibleResolvable' # create service paths if they don't already exist try: foo = DbusIf.DbusService[self.installedVersionPath] except: DbusIf.DbusService.add_path (self.installedVersionPath, "" ) try: foo = DbusIf.DbusService[self.gitHubVersionPath] except: DbusIf.DbusService.add_path (self.gitHubVersionPath, "" ) try: foo = DbusIf.DbusService[self.packageVersionPath] except: DbusIf.DbusService.add_path (self.packageVersionPath, "" ) try: foo = DbusIf.DbusService[self.incompatiblePath] except: DbusIf.DbusService.add_path (self.incompatiblePath, "" ) try: foo = DbusIf.DbusService[self.incompatibleDetailsPath] except: DbusIf.DbusService.add_path (self.incompatibleDetailsPath, "" ) try: foo = DbusIf.DbusService[self.IncompatibleResolvablePath] except: DbusIf.DbusService.add_path (self.IncompatibleResolvablePath, "" ) self.packageNamePath = '/Settings/PackageManager/' + section + '/PackageName' self.gitHubUserPath = '/Settings/PackageManager/' + section + '/GitHubUser' self.gitHubBranchPath = '/Settings/PackageManager/' + section + '/GitHubBranch' # temporarily set PackageName since settingChangeHandler may be called as soon as SettingsDevice is called # which is before actual package name is set below # so this avoids a crash self.PackageName = "" settingsList = {'packageName': [ self.packageNamePath, '', 0, 0 ], 'gitHubUser': [ self.gitHubUserPath, '', 0, 0 ], 'gitHubBranch': [ self.gitHubBranchPath, '', 0, 0 ], } self.DbusSettings = SettingsDevice(bus=dbus.SystemBus(), supportedSettings=settingsList, eventCallback=self.settingChangedHandler, timeout = 10) # if packageName specified on init, use that name if packageName != None: self.DbusSettings['packageName'] = packageName self.PackageName = packageName # otherwise pull name from dBus Settings # this happens when adding a package when it is already in dbus else: self.PackageName = self.DbusSettings['packageName'] self.GitHubUser = self.DbusSettings['gitHubUser'] self.GitHubBranch = self.DbusSettings['gitHubBranch'] # these flags are used to insure multiple actions aren't executed on top of each other self.DownloadPending = False self.InstallPending = False self.InstallAfterDownload = False # used by ResolveConflicts when doing both download and install self.AutoInstallOk = False self.DependencyErrors = [] self.FileConflicts = [] self.LastPatchErrorUpdate = 0 self.ConflictsResolvable = True self.ActionNeeded = '' self.lastScriptPrecheck = 0 self.lastGitHubRefresh = 0 # init dbus parameters # these have local values also to speed access # only create service parameters for real packages if section != 'Edit': self.SetInstalledVersion ("") self.SetPackageVersion ("") self.SetGitHubVersion ("") self.SetIncompatible ("") # copy dbus info to local values self.GitHubUser = self.DbusSettings['gitHubUser'] self.GitHubBranch = self.DbusSettings['gitHubBranch'] # init edit GitHub info else: self.SetGitHubUser ("?") self.SetGitHubBranch ("?") # dbus Settings is the primary non-volatile storage for packageManager # upon startup, PackageList [] is empty and we need to populate it # from previous dBus Settings in /Settings/PackageManager/... # this is a special case that can't use AddPackage below: # we do not want to create any new Settings !! # it should be "safe" to limit the serch to 0 to < packageCount # we also don't specify any parameters other than the section (index) # # NOTE: this method is called before threads are created so do not LOCK # # returns False if couldn't get the package count from dbus # otherwise returns True # no package count on dbus is an error that would prevent continuing # this should never happen since the DbusIf is instantiated before this call # which creates /Count if it does not exist @classmethod def AddPackagesFromDbus (cls): global DbusIf packageCount = DbusIf.GetPackageCount() if packageCount == None: logging.critical ("dbus PackageManager Settings not set up -- can't continue") return False i = 0 while i < packageCount: # no package name tells PackageClas init to pull package name from dbus cls.PackageList.append (PackageClass ( section = i ) ) i += 1 return True # PackageNameValid # checks the package name to see if it is valid # # invalid names contain strings in the rejectStrings # or complete names in rejectNames # or names beginning with '.' # # returns true if name is OK, false if in the reject list rejectStrings = [ "-current", "-latest", "-main", "-test", "-temp", "-debug", "-beta", "-backup1", "-backup2", "-blind", "-0", "-1", "-2", "-3", "-4", "-5", "-6", "-7", "-8", "-9", "ccgx", " " ] rejectNames = [ "conf", "db", "etc", "home", "keys", "log", "lost+found", "setupOptions", "themes", "tmp", "var", "venus", "vrmfilescache" ] @classmethod def PackageNameValid (cls, packageName): if packageName == None or packageName == "": return False if packageName[0] == '.': return False for reject in cls.rejectNames: if reject == packageName: return False for reject in cls.rejectStrings: if reject in packageName: return False return True # AddStoredPackages # add packages stored in /data to the package list # in order to qualify as a package: # must be a directory # name must not contain strings in the reject lists # name must not include any spaces # directory must contain a file named setup # diretory must contain a file named version # first character of version file must be 'v' # name must be unique - that is not match any existing packages # order of validating tests minimizes execution time (determined emperically) # # AddStoredPackages is called from init # and the AddPackages run () loop for background updates @classmethod def AddStoredPackages (cls): global Platform platformIsRaspberryPi = Platform[0:4] == 'Rasp' for packageName in os.listdir ("/data"): if not PackageClass.PackageNameValid (packageName): continue # if package is already in the active list - skip it DbusIf.LOCK ("AddStoredPackages") package = PackageClass.LocatePackage (packageName) DbusIf.UNLOCK ("AddStoredPackages") if package != None: continue packageDir = "/data/" + packageName # skip if no setup file - also verifies packageDir is a directory! if not os.path.exists (packageDir + "/setup"): continue # skip if no version file or not a valid version versionFile = packageDir + "/version" try: fd = open (versionFile, 'r') version = fd.readline().strip() fd.close () except: continue if version == "" or version[0] != 'v': continue # skip if package is for Raspberry PI only and platform is not if os.path.exists (packageDir + "/raspberryPiOnly") and not platformIsRaspberryPi: continue # skip if package was manually removed if not PackageClass.GetAutoAddOk (packageName): continue # package is unique and passed all tests - schedule the package addition PushAction ( command='add:' + packageName, source='AUTO') # AddPackage adds one package to the package list # packageName must be specified # the package names must be unique # # this method is called from the GUI add package command @classmethod def AddPackage ( cls, packageName=None, gitHubUser=None, gitHubBranch=None, source=None ): if source == 'GUI': reportStatusTo = 'Editor' # AUTO or DEFAULT source else: reportStatusTo = None if packageName == None or packageName == "": DbusIf.UpdateStatus ( message="no package name for AddPackage - nothing done", where=reportStatusTo, logLevel=ERROR ) if source == 'GUI': DbusIf.AcknowledgeGuiEditAction ( 'ERROR' ) return False # insure packageName is unique before adding this new package success = False DbusIf.LOCK ("AddPackage") package = PackageClass.LocatePackage (packageName) # new packageName is unique, OK to add it if package == None: DbusIf.UpdateStatus ( message="adding " + packageName, where='Editor', logLevel=INFO ) section = len(cls.PackageList) cls.PackageList.append( PackageClass ( section, packageName = packageName ) ) DbusIf.UpdatePackageCount () success = True # add user/branch from caller package = PackageClass.LocatePackage (packageName) if package != None: if gitHubUser == None: gitHubUser = "?" if gitHubBranch == None: gitHubBranch = "?" package.SetGitHubUser (gitHubUser) package.SetGitHubBranch (gitHubBranch) if source == 'GUI': DbusIf.AcknowledgeGuiEditAction ( '' ) DbusIf.UpdateStatus ( message = "", where='Editor') # allow auto adds and auto installs PackageClass.SetAutoAddOk (packageName, True) package.SetAutoInstallOk (True) else: if source == 'GUI': DbusIf.UpdateStatus ( message=packageName + " already exists - choose another name", where=reportStatusTo, logLevel=INFO ) DbusIf.AcknowledgeGuiEditAction ( 'ERROR' ) else: DbusIf.UpdateStatus ( message=packageName + " already exists", where=reportStatusTo, logLevel=WARNING ) DbusIf.UNLOCK ("AddPackage") return success # end AddPackage # packages are removed as a request from the GUI (packageName specified) # or during system initialization (packageIndex specified) # to remove a package: # 1) locate the entry matching package name (if any) # 2) move all packages after that entry to the previous slot (if any) # 3) erase the last package slot to avoid confusion (by looking at dbus-spy) # 3) remove the entry in PackageList (pop) # 4) update the package count # 5) set DO_NOT_AUTO_ADD flag file to prevent # package from being re-added to the package list # flag file is deleted when package is manually installed again # # Remove package must be passed either the package name or an index into PackageList # # returns True if package was removed, False if not # # this is all done while the package list is locked !!!! @classmethod def RemovePackage (cls, packageName=None, packageIndex=None, isDuplicate=False ): # packageName specified so this is a call from the GUI if packageName != None: guiRequestedRemove = True if packageName == "SetupHelper": DbusIf.UpdateStatus ( message="REMOVING SetupHelper" + packageName, where='Editor', logLevel=CRITICAL ) else: DbusIf.UpdateStatus ( message="removing " + packageName, where='Editor', logLevel=INFO ) # no package name specified, so this is a call from system initialization - messages to log only elif packageIndex != None: guiRequestedRemove = False name = PackageClass.PackageList [packageIndex].PackageName if name == None or name == "": logging.error ( "RemovePackage: removing package without a name" ) else: logging.info ( "RemovePackage: removing " + name ) # neither package name nor package instance passed - can't do anything else: logging.error ( "RemovePackage: no package info passed - nothing done" ) return DbusIf.LOCK ("RemovePackage") packages = PackageClass.PackageList listLength = len (packages) if listLength == 0: DbusIf.UNLOCK ("RemovePackage") return # locate index of packageName # LocaatePackage not used because we want the index anyway if guiRequestedRemove: toIndex = 0 matchFound = False while toIndex < listLength: if packageName == packages[toIndex].PackageName: matchFound = True break toIndex += 1 # called from init - already have index else: toIndex = packageIndex matchFound = True packageIsInstalled = packages[toIndex].InstalledVersion != "" # if package is installed, don't remove it if matchFound and not packageIsInstalled: # if not just removing a duplicate # block future automatic adds since the package is being removed if not isDuplicate: PackageClass.SetAutoAddOk (packageName, False) # move packages after the one to be remove down one slot (copy info) # each copy overwrites the lower numbered package fromIndex = toIndex + 1 while fromIndex < listLength: # dbus Settings toPackage = packages[toIndex] fromPackage = packages[fromIndex] toPackage.SetPackageName (fromPackage.PackageName ) toPackage.SetGitHubUser (fromPackage.GitHubUser ) toPackage.SetGitHubBranch (fromPackage.GitHubBranch ) # dbus service params toPackage.SetGitHubVersion (fromPackage.GitHubVersion ) toPackage.SetPackageVersion (fromPackage.PackageVersion ) toPackage.SetInstalledVersion (fromPackage.InstalledVersion ) toPackage.SetIncompatible (fromPackage.Incompatible, fromPackage.IncompatibleDetails, fromPackage.IncompatibleResolvable ) # package variables toPackage.DownloadPending = fromPackage.DownloadPending toPackage.InstallPending = fromPackage.InstallPending toPackage.AutoInstallOk = fromPackage.AutoInstallOk toPackage.DependencyErrors = fromPackage.DependencyErrors toPackage.FileConflicts = fromPackage.FileConflicts toPackage.LastPatchErrorUpdate = fromPackage.LastPatchErrorUpdate toPackage.lastScriptPrecheck = fromPackage.lastScriptPrecheck toPackage.lastGitHubRefresh = fromPackage.lastGitHubRefresh toPackage.ActionNeeded = fromPackage.ActionNeeded toIndex += 1 fromIndex += 1 # here, toIndex points to the last package in the old list toPackage = packages[toIndex] # can't actually remove service paths cleanly # so just set contents to null/False # they will disappear after PackageManager is started the next time toPackage.SetGitHubVersion ("?") toPackage.SetInstalledVersion ("?") toPackage.SetPackageVersion ("?") toPackage.SetIncompatible ("") toPackage.LastPatchErrorUpdate = 0 toPackage.lastScriptPrecheck = 0 toPackage.lastGitHubRefresh = 0 toPackage.ActionNeeded = NONE # remove the Settings and service paths for the package being removed DbusIf.RemoveDbusSettings ( [toPackage.packageNamePath, toPackage.gitHubUserPath, toPackage.gitHubBranchPath] ) # remove entry from package list packages.pop (toIndex) DbusIf.UpdatePackageCount () DbusIf.UNLOCK ("RemovePackage") # this package was manually removed so block automatic adds # in the package directory if guiRequestedRemove: if matchFound: # block automatic adds PackageClass.SetAutoAddOk (packageName, False) DbusIf.UpdateStatus ( message="", where='Editor' ) DbusIf.AcknowledgeGuiEditAction ( '' ) else: DbusIf.UpdateStatus ( message=packageName + " not removed - name not found", where='Editor', logLevel=ERROR ) DbusIf.AcknowledgeGuiEditAction ( 'ERROR' ) return matchFound # end RemovePackage # UpdateVersionsAndFlags # # retrieves packages versions from the file system # each package contains a file named version in it's root directory # that becomes packageVersion # the installedVersion-... file is associated with installed packages # abesense of the file indicates the package is not installed # presense of the file indicates the package is installed # the content of the file is the actual version installed # in prevous versions of the setup scripts, this file could be empty, # so we show this as "unknown" # # also sets incompatible parameter and AutoInstallOk local variable to save time in other loops # # the single package variation is broken out so it can be called from other methods # to insure version information is up to date before proceeding with an operaiton # # must be called while LOCKED !! def UpdateVersionsAndFlags (self, doConflictChecks=False, doScriptPreChecks=False): global VersionToNumber global VenusVersion global VenusVersionNumber global Platform packageName = self.PackageName # fetch installed version installedVersionFile = "/etc/venus/installedVersion-" + packageName try: versionFile = open (installedVersionFile, 'r') except: installedVersion = "" else: installedVersion = versionFile.readline().strip() versionFile.close() # if file is empty, an unknown version is installed if installedVersion == "": installedVersion = "unknown" self.SetInstalledVersion (installedVersion) packageDir = "/data/" + packageName # no package directory - null out all params if not os.path.isdir (packageDir): self.SetPackageVersion ("") self.AutoInstallOk = False self.SetIncompatible ("no package") return # fetch package version (the one in /data/packageName) try: versionFile = open (packageDir + "/version", 'r') packageVersion = versionFile.readline().strip() versionFile.close() except: packageVersion = "" self.SetPackageVersion (packageVersion) compatible = True # set the incompatible parameter # to 'PLATFORM' or 'VERSION' if os.path.exists (packageDir + "/raspberryPiOnly" ): if Platform[0:4] != 'Rasp': self.SetIncompatible ("incompatible with " + Platform) compatible = False doConflictChecks = False # update local auto install flag based on DO_NOT_AUTO_INSTALL flagFile = "/data/setupOptions/" + packageName + "/DO_NOT_AUTO_INSTALL" if os.path.exists (flagFile): self.AutoInstallOk = False else: self.AutoInstallOk = True # platform is OK, now check versions if compatible: try: fd = open (packageDir + "/firstCompatibleVersion", 'r') firstVersion = fd.readline().strip() fd.close () except: firstVersion = "v2.71" try: fd = open (packageDir + "/obsoleteVersion", 'r') obsoleteVersion = fd.readline().strip() fd.close () except: obsoleteVersion = "v9999.9999.9999" firstVersionNumber = VersionToNumber (firstVersion) obsoleteVersionNumber = VersionToNumber (obsoleteVersion) if VenusVersionNumber < firstVersionNumber or VenusVersionNumber >= obsoleteVersionNumber: self.SetIncompatible ("incompatible with " + VenusVersion) compatible = False doConflictChecks = False elif os.path.exists (packageDir + "/validFirmwareVersions"): with open(packageDir + "/validFirmwareVersions") as f: lines = f.readlines () versionPresent = False for line in lines: if line.strip() == VenusVersion: versionPresent = True break if not versionPresent: self.SetIncompatible ("incompatible with " + VenusVersion) compatible = False doConflictChecks = False # check to see if command line is needed for install # the optionsRequired flag in the package directory indicates options must be set before a blind install # the optionsSet flag indicates the options HAVE been set already # so if optionsRequired == True and optionsSet == False, can't install from GUI if compatible: if os.path.exists ("/data/" + packageName + "/optionsRequired" ): if not os.path.exists ( "/data/setupOptions/" + packageName + "/optionsSet"): self.SetIncompatible ("install from command line" ) compatible = False doConflictChecks = False # check to see if file set has errors if compatible: fileSetsDir = packageDir + "/FileSets" fileSet = fileSetsDir + "/" + VenusVersion if os.path.exists (fileSet + "/INCOMPLETE"): self.SetIncompatible ("incomplete file set for " + str (VenusVersion) ) compatible = False doConflictChecks = False # check for package conflicts - but not if an operation is in progress if doConflictChecks and not self.InstallPending and not self.DownloadPending: # update dependencies dependencyFile = "/data/" + packageName + "/packageDependencies" dependencyErrors = [] if os.path.exists (dependencyFile): try: with open (dependencyFile, 'r') as file: for item in file: parts = item.split () if len (parts) < 2: logging.error ("package dependency " + item + " incomplete") continue dependencyPackage = parts [0] dependencyRequirement = parts [1] installedFile = "/etc/venus/installedVersion-" + dependencyPackage packageIsInstalled = os.path.exists (installedFile) packageMustBeInstalled = dependencyRequirement == "installed" if packageIsInstalled != packageMustBeInstalled: dependencyErrors.append ( (dependencyPackage, dependencyRequirement) ) dependencyErrors.sort() except: pass # log dependency changes if they have changed if dependencyErrors != self.DependencyErrors: self.DependencyErrors = dependencyErrors if len (dependencyErrors) > 0: for dependency in dependencyErrors: (dependencyPackage, dependencyRequirement) = dependency logging.info (packageName + " requires " + dependencyPackage + " to be " + dependencyRequirement) else: logging.info ("dependency conflicts for " + packageName + " have been resolved") # check for file conflicts with prevously installed packages # each line in all file lists are checked to see if the .package contains a different package name # if they differ, a conflict between this and the other package exists # requiring one or the other package to be uninstalled # # patched files are NOT checked because they patch the active file # the setup script with the 'check' option will test the patch file # and report any patch failures fileConflicts = [] fileLists = [ "fileList", "fileListVersionIndependent" ] for fileList in fileLists: path = "/data/" + packageName + "/FileSets/" + fileList if not os.path.exists (path): continue try: with open (path, 'r') as file: # valid entries begin with / and everything after white space is discarded # the result should be a full path to one replacment file for entry in file: entry = entry.strip () if not entry.startswith ("/"): continue replacementFile = entry.split ()[0].strip () packagesList = replacementFile + ".package" if not os.path.exists ( packagesList ) : continue # if a package list for an active file changes, # run script checks again to uncover new or resolved conflicts if os.path.getmtime (packagesList) > self.lastScriptPrecheck: doScriptPreChecks = True try: with open (packagesList, 'r') as plFile: for entry2 in plFile: packageFromList = entry2.strip() # here if previously updated file was from a different package if packageFromList != packageName: file = os.path.basename (replacementFile) fileConflicts.append ( (packageFromList, "uninstalled", file) ) except: pass except: logging.critical ("error while reading file lists for " + packageName) continue conflicts = self.DependencyErrors if fileConflicts != self.FileConflicts: self.FileConflicts = fileConflicts if len (fileConflicts) > 0: for (otherPackage, dependency, file) in fileConflicts: logging.info ("to install " + packageName + ", " + otherPackage + " must not be installed (" + file + ")" ) conflicts.append ( ( otherPackage, dependency ) ) else: logging.info ("file conflicts for " + packageName + " have been resolved") details = "" if len (conflicts) > 0: # eliminate duplicates conflicts = list ( set ( conflicts ) ) resolveOk = True for ( otherPackage, dependency ) in conflicts: if dependency == "uninstalled": details += otherPackage + " must not be installed\n" else: conflictPackage = PackageClass.LocatePackage (otherPackage) if conflictPackage == None: details += otherPackage + " must be installed but not available\n" resolveOk = False elif conflictPackage.PackageVersion != "": details += otherPackage + " must be installed\n" elif conflictPackage.GitHubVersion != "": details += otherPackage + " must be downloaded and installed\n" else: details += otherPackage + " unknown\n" self.SetIncompatible ("package conflict", details, resolvable=resolveOk) compatible = False # make sure script checks are run once at boot # (eg patched errors, but there are others) if self.lastScriptPrecheck == 0: doScriptPreChecks = True self.lastScriptPrecheck = time.time () # end if doConflictChecks # force patch error rebuild if other errors used Incompatible fields if not compatible or self.DownloadPending: self.LastPatchErrorUpdate = 0 # check for and report patch errors if there are no other errors elif compatible and not self.InstallPending and not self.DownloadPending: if os.path.exists (packageDir + "/patchErrors"): compatible = False lastPatchErrorUpdate = os.path.getmtime (packageDir + "/patchErrors") # if patchErrors file has updated, rebuild error list if lastPatchErrorUpdate != self.LastPatchErrorUpdate: self.LastPatchErrorUpdate = lastPatchErrorUpdate details = "" patchCheckErrors = [] with open ( packageDir + "/patchErrors" ) as file: for line in file: patchCheckErrors.append ( line ) patchCheckErrors = list ( set ( patchCheckErrors ) ) if len (patchCheckErrors) > 0: for line in patchCheckErrors: patchFailure = line.strip() details += patchFailure + "\n" logging.warning (packageName + " patch check error: " + patchFailure + " ") else: logging.info (packageName + " patch check reported no errors") self.SetIncompatible ("patch error", details ) else: self.LastPatchErrorUpdate = 0 # if no incompatibilities found, clear incompatible dbus parameters # so the GUI will allow installs if compatible: self.SetIncompatible ("") # run setup script to check for file conflicts (can't be checked here) if doScriptPreChecks and os.path.exists ("/data/" + packageName + "/setup"): PushAction ( command='check' + ':' + packageName, source='AUTO' ) # end UpdateVersionsAndFlags # end Package # UpdateGitHubVersionClass # # downloads the GitHub versions # this work is done in a separate thread so network activity can be spaced out # and isolated from other acvities since they may take a while on slow networks # # a message is used to trigger a priority update for a specific package # this is used when the operator changes GitHub user/branch so the version # updates rapidly # or speed up the refresh rate # a STOP message is used to wake the thread # so that the tread can exit wihout waitin for a potentially long timeout # # background refreshes are triggered by a queue timeout while waiting for messages # refreshes occur rapidly to refresh the GUI and minimize time waiting for the versions # or are spaced out over time such that version information is refreshed every 10 seconds # # Instances: # UpdateGitHubVersion (a separate thread) # # Methods: # updateGitHubVersion # run () # StopThread () # # delay for GitHub version refreshes # slow refresh also controls GitHub version expiration FAST_GITHUB_REFRESH = 0.25 NORMAL_GITHUB_REFRESH = 600.0 # 10 minutes HOURLY_GITHUB_REFRESH = 60.0 * 60.0 DAILY_GITHUB_REFRESH = HOURLY_GITHUB_REFRESH * 24.0 class UpdateGitHubVersionClass (threading.Thread): # updateGitHubVersion # # fetches the GitHub version from the internet and stores it in the package # # this is called from the background thread run () below # # if the wget fails, the GitHub version is set to "" # this will happen if the name, user or branch are not correct # or if there is no internet connection # # the package GitHub version is upated # but the version is also returned to the caller # # Instances: # UpdateGitHubVersion (thread) # # Methods: # updateGitHubVersion # SetPriorityGitHubVersion # run (the thread) def updateGitHubVersion (self, packageName, gitHubUser, gitHubBranch): url = "https://raw.githubusercontent.com/" + gitHubUser + "/" + packageName + "/" + gitHubBranch + "/version" try: proc = subprocess.Popen (["wget", "--timeout=10", "-qO", "-", url], bufsize=-1, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, _ = proc.communicate () stdout = stdout.decode ().strip () returnCode = proc.returncode except: logging.error ("wget for version failed " + packageName) gitHubVersion = "" else: if proc.returncode == 0: gitHubVersion = stdout else: gitHubVersion = "" # locate the package with this name and update it's GitHubVersion # if not in the list discard the information DbusIf.LOCK ("updateGitHubVersion") package = PackageClass.LocatePackage (packageName) if package != None: package.SetGitHubVersion (gitHubVersion) package.lastGitHubRefresh = time.time () DbusIf.UNLOCK ("updateGitHubVersion") return gitHubVersion def __init__(self): threading.Thread.__init__(self) self.GitHubVersionQueue = queue.Queue (maxsize = 50) self.threadRunning = True # package needing immediate update self.priorityPackageName = None # SetPriorityGitHubVersion # pushes a priority package version update onto our queue # # 'local' is a dummy source for the queue pull def SetPriorityGitHubVersion (self, command): self.GitHubVersionQueue.put ( (command, 'local'), block=False ) # UpdateGitHubVersion run () # # updates GitHub versions # GitHub access is spaced out to minimize network traffic # a priority update is pushed onto our que when GitHub info changes # # StopThread () is called to shut down the thread when PackageManager is quitting # "STOP" is pushed on to the queue by StopThread to cause the run thread to detect # detect the stop request immediately # # run () blocks on reading from our queue # the timeout for the queue pull paces the version fetches # normally, the timeout will occur with the queue empty # in which case we update the next GitHub version for the next package # the time between version fetches changes # for the first pass, a shorter delay is used # after all packages have been updated, the delay is increased # the shorter delay is used again when we pull "REFRESH" off the queue # # when the que returns an item, it is checked to see if it is either a # prioirty package to update it's GitHub version # "STOP" - indicating run should return # "REFRESH" - indicating the loop should update all package GitHub versions # this is used when download refresh mode/rates change # "ALL" - same as REFRESH but from the GUI # (uses the saem message path from GUI as the prioirty package update) # when entering the Active packages menu # need to clear the dbus /GuiEditAction # checks the threadRunning flag and returns if it is False, # when run returns, the main method should catch the tread with join () def StopThread (self): self.threadRunning = False self.SetPriorityGitHubVersion ( 'STOP' ) def run (self): global WaitForGitHubVersions gitHubVersionPackageIndex = 0 forcedRefresh = True packageListLength = 0 while self.threadRunning: downloadMode = DbusIf.GetAutoDownloadMode () # do initial refreshes quickly if forcedRefresh: delay = FAST_GITHUB_REFRESH # otherwise set delay to complete scan of all versions in the selected refresh period # this prevents GitHub versions from going undefined if refreshes are happening else: if downloadMode == NORMAL_DOWNLOAD: delay = NORMAL_GITHUB_REFRESH elif downloadMode == HOURLY_DOWNLOAD: delay = HOURLY_GITHUB_REFRESH else: delay = DAILY_GITHUB_REFRESH # this prevents divide by zero - value not actually used if packageListLength != 0: delay /= packageListLength # queue gets STOP and REFRESH commands or priority package name # empty queue signals it's time for a background update # queue timeout is used to pace background updates command = "" source = "" packageName = "" try: (command, source) = self.GitHubVersionQueue.get (timeout = delay) parts = command.split (":") length = len (parts) if length >= 1: command = parts [0] if length >= 2: packageName = parts [1] except queue.Empty: # means get() timed out as expected - not an error # timeout indicates it's time to do a background update pass except: logging.error ("pull from GitHubVersionQueue failed") if command == 'STOP' or self.threadRunning == False: return doUpdate = False packageUpdate = False # the REFRESH command triggers a refresh of all pachage Git Hub versions # background scans in the mainLoop are blocked until the refresh is complete if command == 'REFRESH': gitHubVersionPackageIndex = 0 # hold off other processing until refresh is complete WaitForGitHubVersions = True forcedRefresh = True # guarantee at least one pass even if auto downloads are off # refresh request was received from GUI elif source == 'GUI': # acknowledge command ASAP to minimize time GUI is held off DbusIf.AcknowledgeGuiEditAction ('') if command != 'gitHubScan': logging.error ("incomplete GitHub refresh request from GUI: " + str(parts)) # if GUI is requesting a refresh of all package versions, trigger a one-time refresh # but does not block background scans in mainLoop elif packageName == 'ALL': if not forcedRefresh: gitHubVersionPackageIndex = 0 forcedRefresh = True # refresh is for a spcific package elif packageName != "": packageUpdate = True else: logging.error ("missing name in GitHub refresh request from GUI: " + str(parts)) # package priority update NOT from the GUI elif source == 'local' and packageName != "": packageUpdate = True if packageUpdate: DbusIf.LOCK ("UpdateGitHubVersion run 1") package = PackageClass.LocatePackage (packageName) if package != None: user = package.GitHubUser branch = package.GitHubBranch # always do the update for 'local' source if source != 'GUI': doUpdate = True # for GUI - refresh if no version or last refresh more than 30 seconds ago # prevents unnecessary network traffic when navigating PackageManager menus elif package.GitHubVersion == "" or time.time () > package.lastGitHubRefresh + 30: doUpdate = True else: logging.error ("can't fetch GitHub version - " + packageName + " not in package list") DbusIf.UNLOCK ("UpdateGitHubVersion run 1") doBackground = forcedRefresh or downloadMode != AUTO_DOWNLOADS_OFF # no priority update - do background update if not doUpdate and doBackground: DbusIf.LOCK ("UpdateGitHubVersion run 2") packageListLength = len (PackageClass.PackageList) # empty package list - no refreshes possible if packageListLength == 0: gitHubVersionPackageIndex = 0 # select package to update elif gitHubVersionPackageIndex < packageListLength: package = PackageClass.PackageList[gitHubVersionPackageIndex] packageName = package.PackageName user = package.GitHubUser branch = package.GitHubBranch doUpdate = True gitHubVersionPackageIndex += 1 # reached end of list - all package Git Hub versions have been refreshed if gitHubVersionPackageIndex >= packageListLength: gitHubVersionPackageIndex = 0 # notify the main loop that all versions have been refreshed and # download modes can now be changed if appropriate WaitForGitHubVersions = False forcedRefresh = False DbusIf.UNLOCK ("UpdateGitHubVersion run 2") # do the actual background update outsde the above LOCKED section # since the update requires internet access if doUpdate: self.updateGitHubVersion (packageName, user, branch) # end while self.threadRunning # end UpdateGitHubVersion run () # end UpdateGitHubVersionClass # DownloadGitHubPackagesClass # # downloads packages from GitHub, replacing the existing package # # downloads can take significant time, so they are handled in a separate thread # # the GUI and auto download code (in main loop) push download # actions onto this queue # the thread blocks when the queue is empty # # a STOP command is also pushed onto the queue when PackageManager # is shutting down. This unblocks this thread which immediately # reads threadRunning. If false, run () returns # # Instances: # DownloadGitHub (a separate thread) # # Methods: # GitHubDownload # DownloadVersionCheck # run # StopThread # # the run () thread is only responsible for pacing automatic downloads from the internet # commands are pushed onto the processing queue (PushAction) class DownloadGitHubPackagesClass (threading.Thread): def __init__(self): threading.Thread.__init__(self) self.DownloadQueue = queue.Queue (maxsize = 50) self.threadRunning = True # this method downloads a package from GitHub # it is called from run() below # # download requests are pushed for automatic downloads from mainloop # and also for a manual download triggered from the GUI # # automatic downloads that fail are logged but otherwise not reported def GitHubDownload (self, packageName=None, source=None): if source == 'GUI': where = 'Editor' elif source == 'AUTO': where = 'PmStatus' else: where = None errorMessage = None errorDetails = None downloadError = False if packageName == None or packageName == "": logging.error ("GitHubDownload: no package name specified") downloadError = True if not downloadError: packagePath = "/data/" + packageName tempPackagePath = packagePath + "-temp" DbusIf.LOCK ("GitHubDownload - get GitHub user/branch") package = PackageClass.LocatePackage (packageName) gitHubUser = package.GitHubUser gitHubBranch = package.GitHubBranch DbusIf.UNLOCK ("GitHubDownload - get GitHub user/branch") DbusIf.UpdateStatus ( message="downloading " + packageName, where=where, logLevel=INFO ) tempDirectory = "/data/PmDownloadTemp" if not os.path.exists (tempDirectory): os.mkdir (tempDirectory) # create temp directory specific to this thread tempArchiveFile = tempDirectory + "/temp.tar.gz" # download archive if os.path.exists (tempArchiveFile): os.remove ( tempArchiveFile ) url = "https://github.com/" + gitHubUser + "/" + packageName + "/archive/" + gitHubBranch + ".tar.gz" try: proc = subprocess.Popen ( ['wget', '--timeout=120', '-qO', tempArchiveFile, url ], bufsize=-1, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) _, stderr = proc.communicate() stderr = stderr.decode ().strip () returnCode = proc.returncode except: errorMessage = "could not access archive on GitHub " + packageName downloadError = True else: if returnCode != 0: errorMessage = "could not access " + packageName + ' ' + gitHubUser + ' ' + gitHubBranch + " on GitHub" errorDetails = "returnCode:" + str (returnCode) if stderr != "": errorDetails += " stderr:" + stderr downloadError = True if not downloadError: try: proc = subprocess.Popen ( ['tar', '-xzf', tempArchiveFile, '-C', tempDirectory ], bufsize=-1, stdout=subprocess.PIPE, stderr=subprocess.PIPE) _, stderr = proc.communicate () stderr = stderr.decode ().strip () returnCode = proc.returncode except: errorMessage = "could not unpack " + packageName + ' ' + gitHubUser + ' ' + gitHubBranch downloadError = True else: if returnCode != 0: errorMessage = "unpack failed " + packageName + ' ' + gitHubUser + ' ' + gitHubBranch errorDetails = "stderr: " + stderr downloadError = True if not downloadError: # attempt to locate a directory that contains a version file # the first directory in the tree starting with tempDirectory is returned unpackedPath = LocatePackagePath (tempDirectory) if unpackedPath == None: errorMessage = "no archive path for " + packageName downloadError = True if not downloadError: # move unpacked archive to package location # LOCK this section of code to prevent others # from accessing the directory while it's being updated try: if os.path.exists (tempPackagePath): shutil.rmtree (tempPackagePath, ignore_errors=True) # like rm -rf except: pass DbusIf.LOCK ("GitHubDownload - move package") try: if os.path.exists (packagePath): os.rename (packagePath, tempPackagePath) shutil.move (unpackedPath, packagePath) except: errorMessage = "couldn't update " + packageName downloadError = True DbusIf.UNLOCK ("GitHubDownload - move package") DbusIf.LOCK ("GitHubDownload - update status") package = PackageClass.LocatePackage (packageName) if package != None: installAfter = package.InstallAfterDownload # save install after flag for later, then clear it package.InstallAfterDownload = False package.DownloadPending = False if not downloadError: # update basic flags then request install if installAfter: package.UpdateVersionsAndFlags () logging.info ("install after download requested for " + packageName) PushAction ( command='install' + ':' + packageName, source=source ) # no install after, do full version/flag update else: package.UpdateVersionsAndFlags (doConflictChecks=True, doScriptPreChecks=True) DbusIf.UNLOCK ("GitHubDownload - update status") # report errors / success if errorMessage != None: logging.error (errorMessage) if errorDetails != None: logging.error (errorDetails) DbusIf.UpdateStatus ( message=errorMessage, where=where ) if source == 'GUI': if errorMessage != None: DbusIf.AcknowledgeGuiEditAction ( 'ERROR' ) # don't ack success if there's more to do elif not installAfter: DbusIf.AcknowledgeGuiEditAction ( '' ) # remove any remaining temp directories if os.path.exists (tempPackagePath): shutil.rmtree (tempPackagePath, ignore_errors=True) # like rm -rf if os.path.exists (tempDirectory): shutil.rmtree (tempDirectory, ignore_errors=True) # end GitHubDownload # DownloadVersionCheck # # compares versions to determine if a download is needed # returns: True if a download is needed, False otherwise # must be called with package list LOCKED !! def DownloadVersionCheck (self, package): gitHubUser = package.GitHubUser gitHubBranch = package.GitHubBranch gitHubVersion = package.GitHubVersion packageVersion = package.PackageVersion # versions not initialized yet - don't allow the download if gitHubVersion == None or gitHubVersion == "" or gitHubVersion[0] != 'v' or packageVersion == '?': return False packageVersionNumber = package.PackageVersionNumber gitHubVersionNumber = package.GitHubVersionNumber # if GitHubBranch is a version number, a download is needed if the versions differ if gitHubBranch[0] == 'v': if gitHubVersionNumber != packageVersionNumber: return True else: return False # otherwise the download is needed if the gitHubVersion is newer else: if gitHubVersionNumber > packageVersionNumber: return True else: return False # DownloadGitHub run (the thread) # # StopThread () is called to shut down the thread def StopThread (self): self.threadRunning = False self.DownloadQueue.put ( ('STOP', ''), block=False ) # DownloadGitHub run (the thread) # # downloads packages placed on its queue from # GUI requests # a background loop in mainLoop # # run () checks the threadRunning flag and returns if it is False, # essentially taking the thread off-line # the main method should catch the tread with join () def run (self): while self.threadRunning: # loop forever # process one GUI download request # if there was one, skip auto downloads until next pass try: command = self.DownloadQueue.get () # block forever except: logging.error ("pull from DownloadQueue queue failed") time.sleep (5.0) continue if command[0] == 'STOP' or self.threadRunning == False: return # separate command, source tuple # and separate action and packageName if len (command) >= 2: parts = command[0].split (":") if len (parts) >= 2: action = parts[0].strip () packageName = parts[1].strip () else: logging.error ("DownloadQueue - no action and/or package name - discarding", command) continue source = command[1] else: logging.error ("DownloadQueue - no command and/or source - discarding", command) continue # invalid action for this queue if action != 'download': logging.error ("received invalid command from Install queue: ", command ) continue # do the download here self.GitHubDownload (packageName=packageName, source=source ) # end while True # end run # end DownloadGitHubPackagesClass # InstallPackagesClass # Instances: # InstallPackages (a separate thread) # # Methods: # InstallPackage # ResolveConflicts # run (the thread) # StopThread # run # # runs as a separate thread since the operations can take a long time # and we need to space them to avoid consuming all CPU resources # # packages are automatically installed only # if the autoInstall Setting is active # package version is newer than installed version # or if nothing is installed # # a manual install is performed regardless of versions class InstallPackagesClass (threading.Thread): def __init__(self): threading.Thread.__init__(self) DbusIf.SetPmStatus ("") self.threadRunning = True self.InstallQueue = queue.Queue (maxsize = 10) # InstallPackage # # this method either installs, uninstalls or checks a package # by calling the package's setup script # # the 'check' action runs file set checks without installing the package # this creates a missing file set then sets/clears the INCOMPLETE flag # a missing file set is reported as "no file set" but does not block installs # an INCOMPLETE file set blocks installs # therefore, check attempts to resolve a missing file set # # the operation can take many seconds # i.e., the time it takes to run the package's setup script # # uninstalling SetupHelper is a special case since that action will end PackageManager # so it is deferred until mainLoop detects the request and exits to main # where the actual uninstall occurs def InstallPackage ( self, packageName=None, source=None , action='install' ): # refresh versions, then check to see if an install is possible DbusIf.LOCK ("InstallPackage") package = PackageClass.LocatePackage (packageName) if package == None: logging.error ("InstallPackage: " + packageName + " not in package list") if source == 'GUI': DbusIf.UpdateStatus ( message=packageName + " not in package list", where='Editor' ) DbusIf.AcknowledgeGuiEditAction ( 'ERROR' ) DbusIf.UNLOCK ("InstallPackage error 1") return if source == 'GUI': sendStatusTo = 'Editor' # uninstall sets the uninstall flag file to prevent auto install if action == 'uninstall': package.SetAutoInstallOk (False) logging.info (packageName + " was manually uninstalled - auto install for that package will be skipped") # manual install removes the flag file elif action == 'install': package.SetAutoInstallOk (True) logging.info (packageName + " was manually installed - allowing auto install for that package") elif source == 'AUTO': sendStatusTo = 'PmStatus' packageDir = "/data/" + packageName if not os.path.isdir (packageDir): errorMessage = "no package directory " + packageName logging.error ("InstallPackage - " + errorMessage) package.InstallPending = False package.UpdateVersionsAndFlags () if source == 'GUI': DbusIf.AcknowledgeGuiEditAction ( 'ERROR' ) DbusIf.UNLOCK ("InstallPackage error 2") return setupFile = packageDir + "/setup" if not os.path.isfile(setupFile): errorMessage = "setup file for " + packageName + " doesn't exist" DbusIf.UpdateStatus ( message=errorMessage, where=sendStatusTo, logLevel=ERROR ) package.InstallPending = False package.UpdateVersionsAndFlags () if source == 'GUI': DbusIf.AcknowledgeGuiEditAction ( 'ERROR' ) DbusIf.UNLOCK ("InstallPackage error 3") return elif os.access(setupFile, os.X_OK) == False: errorMessage = "setup file for " + packageName + " not executable" DbusIf.UpdateStatus ( message=errorMessage, where=sendStatusTo, logLevel=ERROR ) package.InstallPending = False if source == 'GUI': DbusIf.AcknowledgeGuiEditAction ( 'ERROR' ) DbusIf.UNLOCK ("InstallPackage error 4") return DbusIf.UNLOCK ("InstallPackage normal") DbusIf.UpdateStatus ( message=action + "ing " + packageName, where=sendStatusTo, logLevel=INFO ) try: proc = subprocess.Popen ( [ setupFile, action, 'runFromPm' ], bufsize=100000, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True ) proc.wait () # forward stdout lines from setup script to console/log file for line in proc.stdout: logging.info ( line.strip () ) # collect stderr lines for possible use later stderr = "" for line in proc.stderr: stderr += line returnCode = proc.returncode setupRunFail = False except: setupRunFail = True # manage the result of the setup run while locked just in case DbusIf.LOCK ("InstallPackage - update status") package = PackageClass.LocatePackage (packageName) package.InstallPending = False errorMessage = "" if setupRunFail: errorMessage = "could not run setup" elif returnCode == EXIT_SUCCESS: DbusIf.UpdateStatus ( message="", where=sendStatusTo ) if source == 'GUI': DbusIf.AcknowledgeGuiEditAction ( '' ) elif returnCode == EXIT_REBOOT: package.ActionNeeded = REBOOT_NEEDED if source == 'GUI': logging.info ( packageName + " " + action + " REBOOT needed but handled by GUI") DbusIf.UpdateStatus ( message="", where=sendStatusTo ) DbusIf.AcknowledgeGuiEditAction ( "" ) # auto install triggers a reboot by setting the global flag - reboot handled in main_loop else: logging.info ( packageName + " " + action + " REBOOT pending") global SystemReboot SystemReboot = True elif returnCode == EXIT_RESTART_GUI: package.ActionNeeded = GUI_RESTART_NEEDED if source == 'GUI': logging.info ( packageName + " " + action + " GUI restart needed but handled by GUI") DbusIf.UpdateStatus ( message="", where=sendStatusTo ) DbusIf.AcknowledgeGuiEditAction ( "" ) # auto install triggers a GUI restart by setting the global flag - restart handled in main_loop else: logging.info ( packageName + " " + action + " GUI restart pending") global GuiRestart GuiRestart = True elif returnCode == EXIT_RUN_AGAIN: if source == 'GUI': DbusIf.UpdateStatus ( message=packageName + " run install again to complete install", where=sendStatusTo, logLevel=INFO ) DbusIf.AcknowledgeGuiEditAction ( 'ERROR' ) else: DbusIf.UpdateStatus ( message=packageName + " setup must be run again", where=sendStatusTo, logLevel=WARNING ) elif returnCode == EXIT_INCOMPATIBLE_VERSION: global VenusVersion errorMessage = "incompatible with " + VenusVersion elif returnCode == EXIT_INCOMPATIBLE_PLATFORM: global Platform errorMessage = "incompatible with " + Platform elif returnCode == EXIT_OPTIONS_NOT_SET: errorMessage = "setup must be run from the command line" elif returnCode == EXIT_FILE_SET_ERROR: errorMessage = "incomplete file set for " + VenusVersion elif returnCode == EXIT_ROOT_FULL: errorMessage = "no room on root partition " elif returnCode == EXIT_DATA_FULL: errorMessage = "no room on data partition " elif returnCode == EXIT_NO_GUI_V1: errorMessage = "failed - " + "GUI v1 not installed" elif returnCode == EXIT_PACKAGE_CONFLICT: errorMessage = "package conflict " + stderr elif returnCode == EXIT_PATCH_ERROR: errorMessage = "could not patch some files" # unknown error elif returnCode != 0: errorMessage = "unknown error " + str (returnCode) + " " + stderr if errorMessage != "": if setupRunFail: logLevel = ERROR else: logLevel = INFO DbusIf.UpdateStatus ( message=packageName + " " + action + " failed - " + errorMessage, where=sendStatusTo, logLevel=logLevel ) if source == 'GUI': DbusIf.AcknowledgeGuiEditAction ( 'ERROR' ) # installs do script conflict checks # update last check time here so checks aren't run right away package.lastScriptPrecheck = time.time () package.UpdateVersionsAndFlags () DbusIf.UNLOCK ("InstallPackage - update status") # end InstallPackage () # ResolveConflicts # # this method checks the conflicts for the indicated package # if conflicts exist, the conflicting package(s) are installed or uninstalled # by pushing them on the install queue def ResolveConflicts ( self, packageName=None, source=None ): if packageName == None: logging.error ("ResolveConflicts - no package name specified") return DbusIf.LOCK ("ResolveConflicts") package = PackageClass.LocatePackage (packageName) if package == None: logging.error ("ResolveConflicts: " + packageName + "not found") for conflict in (package.DependencyErrors + package.FileConflicts): if len (conflict) < 2: logging.error ("ResolveConflicts: " + packageName + " missing parameters: " + str (conflict) ) continue dependencyPackage = conflict[0] dependencyRequirement = conflict[1] if dependencyRequirement == "installed": packageMustBeInstalled = True elif dependencyRequirement == "uninstalled": packageMustBeInstalled = False else: logging.error ("ResolveConflicts: " + packageName + " unrecognized requirement: " + str (conflict) ) continue requiredPackage = PackageClass.LocatePackage (dependencyPackage) if requiredPackage.InstalledVersion != "": packageIsInstalled = True else: packageIsInstalled = False if requiredPackage.PackageVersion != "": packageIsStored = True else: packageIsStored = False if requiredPackage.GitHubVersion != "": packageIsOnGitHub = True else: packageIsOnGitHub = False if packageIsStored or packageIsOnGitHub: packageIsAvailable = True else: packageIsAvailable = True if packageMustBeInstalled and not packageIsInstalled: if not packageIsAvailable: DbusIf.UpdateStatus ( message=dependencyPackage + " not available - can't install", where='Editor', logLevel=WARNING ) elif not packageIsStored and packageIsOnGitHub: logging.info ("ResolveConflicts: downloading and installing" + dependencyPackage + " so that " + packageName + " can be installed" ) PushAction ( command='download' + ':' + dependencyPackage, source=source ) # download will trigger install when it finished requiredPackage.InstallAfterDownload = True else: logging.info ("ResolveConflicts: installing " + dependencyPackage + " so that " + packageName + " can be installed" ) PushAction ( command='install' + ':' + dependencyPackage, source=source ) elif not packageMustBeInstalled and packageIsInstalled: logging.info ("ResolveConflicts: uninstalling " + dependencyPackage + " so that " + packageName + " can be installed" ) PushAction ( command='uninstall' + ':' + dependencyPackage, source=source ) DbusIf.UNLOCK ("ResolveConflicts") # InstallPackage run (the thread) # # automatic install packages # pushes request on queue for processing later in another thread # this allows this to run quickly while the package list is locked # # run () checks the threadRunning flag and returns if it is False, # essentially taking the thread off-line # the main method should catch the tread with join () # StopThread () is called to shut down the thread def StopThread (self): self.threadRunning = False self.InstallQueue.put ( ('STOP', ''), block=False ) def run (self): while self.threadRunning: try: command = self.InstallQueue.get () except: logging.error ("pull from Install queue failed") continue if len (command) == 0: logging.error ("pull from Install queue failed - empty comand") continue # thread shutting down if command[0] == 'STOP' or self.threadRunning == False: return # separate command, source tuple # and separate action and packageName if len (command) >= 2: parts = command[0].split (":") if len (parts) >= 2: action = parts[0].strip () packageName = parts[1].strip () else: logging.error ("InstallQueue - no action and/or package name - discarding", command) continue source = command[1] else: logging.error ("InstallQueue - no command and/or source - discarding", command) continue # resolve conflicts may cause OTHER packages to install or uninstall if action == 'resolveConflicts': self.ResolveConflicts (packageName=packageName, source=source) # otherwise use InstallPackage to install, uninstall, or check the package elif action == 'install' or action == 'uninstall' or action == 'check': self.InstallPackage (packageName=packageName, source=source , action=action ) # invalid action for this queue else: logging.error ("received invalid command from Install queue: ", command ) continue # end run # end InstallPackagesClass # MediaScanClass # Instances: # MediaScan (a separate thread) # Methods: # transferPackage # StopThread # settingsBackup # settingsRestore # run # # scan removable SD and USB media for packages to be installed # # run () is a separate thread that looks for removable # SD cards and USB sticks that appear in /media as separate directories # these directories come and go with the insertion and removable of the media # # when new media is detected, it is scanned once then ignored # when media is removed, then reinserted, the scan begins again # # packages must be located in the root of the media (no subdirecoties are scanned) # and must be an archive with a name ending in .tar.gz # # archives are unpacked to a temp directory in /var/run (a ram disk) # verified, then moved into position in /data/ # where the name comes from the unpacked directory name # of the form - # # actual installation is handled in the InstallPackages run() thread # # removable media is checked for several "flag files" that trigger actions elsewhere # these are described in detail at the beginning of this file # scans for flag files is done in run () class MediaScanClass (threading.Thread): # transferPackage unpacks the archive and moves it into postion in /data # # path is the full path to the archive # # if true, autoInstallOverride causes the ONE_TIME_INSTALL flag to be set # this happens if the caller detects the AUTO_INSTALL_PACKAGES flag on removable media def transferPackage (self, path, autoInstallOverride=False): packageName = os.path.basename (path).split ('-', 1)[0] # create an empty temp directory in ram disk # for the following operations # directory is unique to this process and thread tempDirectory = "/var/run/packageManager" + str(os.getpid ()) + "Media" if os.path.exists (tempDirectory): shutil.rmtree (tempDirectory) os.mkdir (tempDirectory) # unpack the archive - result is placed in tempDirectory try: proc = subprocess.Popen ( ['tar', '-xzf', path, '-C', tempDirectory ], bufsize=-1, stdout=subprocess.PIPE, stderr=subprocess.PIPE) _, stderr = proc.communicate () stderr = stderr.decode ().strip () returnCode = proc.returncode except: DbusIf.UpdateStatus ( message="tar failed for " + packageName, where='Media', logLevel=ERROR) time.sleep (5.0) DbusIf.UpdateStatus ( message="", where='Media') return False if returnCode != 0: DbusIf.UpdateStatus ( message="could not unpack " + packageName + " from SD/USB media", where='Media', logLevel=ERROR) logging.error ("stderr: " + stderr) shutil.rmtree (tempDirectory) time.sleep (5.0) DbusIf.UpdateStatus ( message="", where='Media') return False # attempt to locate a package directory in the tree below tempDirectory unpackedPath = LocatePackagePath (tempDirectory) if unpackedPath == None: logging.warning (packageName + " archive doesn't contain a package directory - rejected" ) shutil.rmtree (tempDirectory) time.sleep (5.0) DbusIf.UpdateStatus ( message="", where='Media') return False # compare versions and proceed only if they are different packagePath = "/data/" + packageName try: fd = open (packagePath + "/version", 'r') except: packageVersion = 0 else: packageVersion = VersionToNumber (fd.readline().strip()) fd.close () try: fd = open (unpackedPath + "/version", 'r') except: unpackedVersion = 0 else: unpackedVersion = VersionToNumber (fd.readline().strip()) fd.close () if packageVersion == unpackedVersion: logging.info ("transferPackages: " + packageName + " versions are the same - skipping transfer") shutil.rmtree (tempDirectory) DbusIf.UpdateStatus ( message="", where='Media') return False # move unpacked archive to package location # LOCK this critical section of code to prevent others # from accessing the directory while it's being updated DbusIf.UpdateStatus ( message="transfering " + packageName + " from SD/USB", where='Media', logLevel=INFO ) tempPackagePath = packagePath + "-temp" DbusIf.LOCK ("transferPackage") if os.path.exists (tempPackagePath): shutil.rmtree (tempPackagePath, ignore_errors=True) # like rm -rf if os.path.exists (packagePath): os.rename (packagePath, tempPackagePath) try: shutil.move (unpackedPath, packagePath) except: logging.error ( "transferPackages: couldn't relocate " + packageName ) if os.path.exists (tempPackagePath): shutil.rmtree (tempPackagePath, ignore_errors=True) # like rm -rf # set package one-time install flag so this package is installed regardless of other flags # this flag is removed when the install is preformed if autoInstallOverride: logging.warning ("Auto Install - setting ONE_TIME_INSTALL for " + packageName ) open ( packagePath + "/ONE_TIME_INSTALL", 'a').close() DbusIf.UNLOCK ("transferPackage") shutil.rmtree (tempDirectory, ignore_errors=True) time.sleep (5.0) DbusIf.UpdateStatus ( message="", where='Media') return True # end transferPackage def __init__(self): threading.Thread.__init__(self) self.MediaQueue = queue.Queue (maxsize = 10) # used only for STOP self.threadRunning = True self.AutoUninstall = False # # settingsBackup # settingsRestore # # extracts / restores dbus settings and custom icons # copies ALL log files (backup only obvously) # the files are zipped to save space # # backup and restore options are either to/from removable media or /data # # settingsList contains the list of dbus /Settings parameters to save and restore # def settingsBackup (self, backupPath, settingsOnly = False): settingsCount = 0 overlayCount = 0 logsWritten = "no logs" settingsListFile = "/data/SetupHelper/settingsList" backupFile = backupPath + "/settingsBackup" try: if not os.path.exists (settingsListFile): logging.error (settingsListFile + " does not exist - can't backup settings") return # backup settings backupSettings = open (backupFile, 'w') bus = dbus.SystemBus() with open (settingsListFile, 'r') as listFile: for line in listFile: setting = line.strip() try: value = bus.get_object("com.victronenergy.settings", setting).GetValue() attributes = bus.get_object("com.victronenergy.settings", setting).GetAttributes() except: continue dataType = type (value) if dataType is dbus.Double: typeId = 'f' elif dataType is dbus.Int32 or dataType is dbus.Int64: typeId = 'i' elif dataType is dbus.String: typeId = 's' else: typeId = '' logging.error ("settingsBackup - invalid data type " + typeId + " - can't include parameter attributes " + setting) value = str ( value ) default = str (attributes[0]) min = str (attributes[1]) max = str (attributes[2]) silent = str (attributes[3]) # create entry with just settng path and value without a valid data type if typeId == '': line = ','.join ( [ setting, value ]) + '\n' else: line = ','.join ( [ setting, value, typeId, default, min, max, silent ]) + '\n' backupSettings.write (line) settingsCount += 1 backupSettings.close () listFile.close () except: logging.error ("settings backup - settings write failure") if not settingsOnly: # backup logo overlays overlaySourceDir = "/data/themes/overlay" overlayDestDir = backupPath + "/logoBackup" # remove any previous logo backups if os.path.isdir (overlayDestDir): shutil.rmtree (overlayDestDir) try: if os.path.isdir (overlaySourceDir): overlayFiles = os.listdir (overlaySourceDir) if len (overlayFiles) > 0: # create overlay direcory on backkup device, then copy files if not os.path.isdir (overlayDestDir): os.mkdir (overlayDestDir) for overlay in overlayFiles: if overlay[0] == ".": continue shutil.copy ( overlaySourceDir + "/" + overlay, overlayDestDir ) overlayCount += 1 except: logging.error ("settings backup - logo write failure") # copy log files try: # remove any previous log backups logDestDir = backupPath + "/logs" if os.path.isdir (logDestDir): shutil.rmtree (logDestDir) proc = subprocess.Popen ( [ 'zip', '-rq', backupPath + "/logs.zip", "/data/log" ], bufsize=-1, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) proc.commiunicate() #output ignored returnCode = proc.returncode logsWritten = "logs" except: logging.error ("settings backup - log write failure") logsWritten = "no logs" # backup setup script options optionsSourceDir = "/data/setupOptions" optionsDestDir = backupPath + "/setupOptions" try: # remove any previous options backups if os.path.isdir (optionsDestDir): shutil.rmtree (optionsDestDir) if os.path.isdir (optionsSourceDir): shutil.copytree ( optionsSourceDir, optionsDestDir ) except: logging.error ("settings backup - overlays write failure") logging.info ("settings backup completed - " + str(settingsCount) + " settings, " + str (overlayCount) + " logos, " + logsWritten ) def settingsRestore (self, backupPath, settingsOnly = False): backupFile = backupPath + "/settingsBackup" if not os.path.exists (backupFile): logging.error (backupFile + " does not exist - can't restore settings") bus = dbus.SystemBus() settingsCount = 0 overlayCount = 0 with open (backupFile, 'r') as fd: for line in fd: parameterExists = False # ( setting path, value, attributes) parts = line.strip().split (',') numberOfParts = len (parts) # full entry with attributes if numberOfParts == 7: typeId = parts[2] default = parts[3] min = parts[4] max = parts[5] silent = parts[6] # only path and name - old settings file format elif numberOfParts == 2: typeId = '' default = '' min = '' max = '' silent = '' else: logging.error ("settingsRestore: invalid line in file " + line) continue path = parts[0] value = parts[1] try: bus.get_object("com.victronenergy.settings", path).GetValue() parameterExists = True except: pass if not parameterExists: if typeId == '': logging.error ("settingsRestore: no attributes in settingsBackup file - can't create " + path) # parameter does not yet exist, create it else: # silent uses a different method if silent == 1: method = 'AddSettingSilent' else: method = 'AddSetting' try: proc = subprocess.Popen ( [ 'dbus', '-y', 'com.victronenergy.settings', '/', method, '', path, default, typeId, min, max ], bufsize=-1, stdout=subprocess.PIPE, stderr=subprocess.PIPE) proc.commiunicate () # output ignored parameterExists = True logging.info ("settingsRestore: creating " + path) except: logging.error ("settingsRestore: settings create failed for " + path) # update parameter's value if it exists (or was just created) if parameterExists: bus.get_object("com.victronenergy.settings", path).SetValue (value) settingsCount += 1 if not settingsOnly: # restore logo overlays overlaySourceDir = backupPath + "/logoBackup" overlayDestDir = "/data/themes/overlay" if os.path.isdir (overlaySourceDir): overlayFiles = os.listdir (overlaySourceDir) if len (overlayFiles) > 0: # create overlay direcory in /data, then copy files if not os.path.isdir (overlayDestDir): os.mkdir (overlayDestDir) for overlay in overlayFiles: if overlay[0] == ".": continue try: shutil.copy ( overlaySourceDir + "/" + overlay, overlayDestDir ) except: logging.error ("settingsRestore: overlay create failed for " + overlay) overlayCount += 1 # restore setup script options optionsSourceDir = backupPath + "/setupOptions" optionsDestDir = "/data/setupOptions" # remove any previous options backups if os.path.isdir (optionsDestDir): shutil.rmtree (optionsDestDir) if os.path.isdir (optionsSourceDir): try: shutil.copytree ( optionsSourceDir, optionsDestDir ) except: logging.error ("settingsRestore: options restore failed") logging.info ("settings restore completed - " + str(settingsCount) + " settings and " + str (overlayCount) + " overlays") # Media Scan run (the thread) # # run () checks the threadRunning flag and returns if it is False, # essentially taking the thread off-line # the main method should catch the tread with join () # StopThread () is called to shut down the thread # # a queue with set timeout is used to pace operations # this gives other threads time away from slower media scanning operations def StopThread (self): self.threadRunning = False self.MediaQueue.put ( "STOP", block=False ) def run (self): separator = '/' root = "/media" archiveSuffix = ".tar.gz" autoRestore = False autoRestoreComplete = False autoEject = False bus = dbus.SystemBus() localSettingsBackupExists = True # list of accepted branch/version substrings acceptList = [ "-current", "-latest", "-main", "-test", "-debug", "-beta", "-install", "-0", "-1", "-2", "-3", "-4", "-5", "-6", "-7", "-8", "-9" ] # keep track of all media that's been scanned so it isn't scanned again # media removal removes it from this list alreadyScanned = [] while self.threadRunning: # use queue to receive stop command and also to space operations command = "" try: command = self.MediaQueue.get (timeout = 5.0) except queue.Empty: # queue empty is OK # timeout indicates it's time to make one pass through the code below pass except: logging.error ("pull from MediaQueue failed") time.sleep (5.0) if command == 'STOP' or self.threadRunning == False: return # automaticTransfers is used to signal when anything is AUTOMATICALLY # transferred from or to removable media # this includes: # transfrring a package from removable media to /data # performing an automatic settings restore # Manually triggered operations do not update these operations # manually triggered settings backup # manually triggered settings restore automaticTransfers = False # do local settings backup/restore if os.path.exists ("/data/settingsBackup"): localSettingsBackupExists = True DbusIf.SetBackupSettingsLocalFileExist (True) else: localSettingsBackupExists = False DbusIf.SetBackupSettingsLocalFileExist (False) backupProgress = DbusIf.GetBackupProgress () if backupProgress == 21: DbusIf.SetBackupProgress (23) self.settingsBackup ("/data", settingsOnly = True) DbusIf.SetBackupProgress (0) elif backupProgress == 22: if localSettingsBackupExists: DbusIf.SetBackupProgress (24) self.settingsRestore ("/data", settingsOnly = True) DbusIf.SetBackupProgress (0) try: drives = os.listdir (root) except: drives = [] if len (drives) == 0: DbusIf.SetBackupMediaAvailable (False) DbusIf.SetBackupSettingsFileExist (False) backupMediaExists = False else: DbusIf.SetBackupMediaAvailable (True) backupMediaExists = True # if previously detected media is removed, # allow it to be scanned again when reinserted for scannedDrive in alreadyScanned: if not scannedDrive in drives: alreadyScanned.remove (scannedDrive) for drive in drives: drivePath = separator.join ( [ root, drive ] ) self.AutoUninstall = False # process settings backup and restore # check for settings backup file mediaSettingsBackupPath = root + "/" + drive if os.path.exists (mediaSettingsBackupPath + "/settingsBackup"): DbusIf.SetBackupSettingsFileExist (True) backupSettingsFileExists = True else: DbusIf.SetBackupSettingsFileExist (False) backupSettingsFileExists = False if backupMediaExists: autoRestoreFile = mediaSettingsBackupPath + "/SETTINGS_AUTO_RESTORE" if os.path.exists (autoRestoreFile): autoRestore = True autoEjectFile = mediaSettingsBackupPath + "/AUTO_EJECT" if os.path.exists (autoEjectFile): autoEject = True initializeFile = mediaSettingsBackupPath + "/INITIALIZE_PACKAGE_MANAGER" if os.path.exists (initializeFile): global InitializePackageManager InitializePackageManager = True # set the auto install flag for use elsewhere autoInstallOverride = False autoUnInstallFile = mediaSettingsBackupPath + "/AUTO_UNINSTALL_PACKAGES" if os.path.exists (autoUnInstallFile): self.AutoUninstall = True # set the auto install flag for use elsewhere # auto Uninstall overrides auto install if not self.AutoUninstall: # check for auto install on media autoInstallFile = mediaSettingsBackupPath + "/AUTO_INSTALL_PACKAGES" if os.path.exists (autoInstallFile): autoInstallOverride = True backupProgress = DbusIf.GetBackupProgress () # GUI triggered backup if backupProgress == 1: DbusIf.SetBackupProgress (3) self.settingsBackup (mediaSettingsBackupPath) DbusIf.SetBackupProgress (0) elif backupProgress == 2 or ( autoRestore and not autoRestoreComplete ): if backupSettingsFileExists: DbusIf.SetBackupProgress (4) self.settingsRestore (mediaSettingsBackupPath) if autoRestore: autoRestoreComplete = True automaticTransfers = True DbusIf.SetBackupProgress (0) # if we've scanned this drive previously, it won't have any new packages to transfer # so skip it to avoid doing it again if drive not in alreadyScanned: # check any file name ending with the achive suffix # all others are skipped for path in glob.iglob (drivePath + "/*" + archiveSuffix): accepted = False if os.path.isdir (path): continue else: accepted = False baseName = os.path.basename (path) # verify the file name contains one of the accepted branch/version identifiers # if not found in the list, the archive is rejected for accept in acceptList: if accept in baseName: accepted = True break # discovered what appears to be a valid archive # unpack it, do further tests and move it to /data if accepted: if self.transferPackage (path, autoInstallOverride): automaticTransfers = True if self.threadRunning == False: return else: logging.warning (path + " not a valid archive name - rejected") # mark this drive so it won't get scanned again # this prevents repeated installs alreadyScanned.append (drive) # end if drive not in alreadyScanned # end for path #end for drive # we have arrived at a point where all removable media has been scanned # and all possible work has been done # eject removable media if work has been done and the # the AUTO_EJECT flag file was fouund on removable media during the most recent scan # NOTE: this ejects ALL removable media whether or not they are involved in transfers if automaticTransfers and autoEject: logging.warning ("automatic media transfers have occured, ejecting ALL removable media") bus.get_object("com.victronenergy.logger", "/Storage/MountState").SetValue (2) if not backupMediaExists: autoRestore = False autoEject = False autoRestoreComplete = False # end while # end run () # end MediaScanClass # main and mainLoop # # Methods: # mainLoop # main ( the entry point) # directUninstall # # mainLoop is called each second to make background checks # update global and package flags, versions, etc # and schedule automatic installs, uninstalls, downloads, etc # # operations that can not be done in line may be deferred here: # GUI restarts and system reboots # PackageManager restarts and INITIALIZE # GUI command acknowledgement from within the thread of the command handler # # handshakes between threads often use a global variable rather than pushing something on a queue: # install/download check holdoff while waiting for GitHub version refresh # # some operations performed in mainLoop take a while but do not need to be done every second # to spread things out, and minimize impact on other tasks within Venus Os, # only one package is checked each second # (it would take 10 seconds to scan 10 packages) # # PackageManager is responsible for reinstalling packages following a firmware update # reinstallMods is a script called from /data/rcS.local that installs the PackageManager service # then sets the /etc/venus/REINSTALL_PACKAGES flag file instructing PackageManager # to do a boot-time check all packages for possible reinstall # PackageManager clears that flag when all packages have been reinstalled # boot-time reinstall is done using the normal automatic install mechanism but bypasses # the test for the user selectable auto install on/off # # mainLoop is resonsible for triggering GUI restarts and system reboots # installs, uninstalls and downloads are handled by the package's setup script # but the script does not initalte these options. # the results of the setup script set reboot/reset flags for the package # the actual restart/reboot actions are held off as long as any package operations are pending # so repeaded GUI restarts or system reboots are avoided # this gives all packages a chance to download and/or install automatically # before the restart/reboot # # mainLoop also handles PackageManager restart and INITIALIZE actions # INITIALZE wipes out the package information in dbus /Settings then quits # The package information is then rebuilt the next time PackageManager starts # # mainLoop also provides status information to the GUI # # GUI restarts are handled without exiting PackageManager, however # system reboots and Package manager restarts and INITIALIZE actions require # PackageManager to exit # The actual operations are performed in main () after mainLoop quits # # mainLoop is "scheduled" to run from GLib which is all set up in main() # however, mainLoop is not a simple loop # it is called once per second by GLib then exits True # mainLoop exits by calling mainloop.quit() then returning False # # main is tasks: # instializing global variables that do not change over time # instantiating objects # retriving nonvolatile information from dbus /Settings # starting all threads # start mainLoop # # the code after mainloop.run() does not execute until mainLoop exits # at which time the activities necessarey for a clean exit are performed: # # an uninstall all function triggered by a flag on removable media # this allows a blind uninstall of all packages (including SetupHelper) # in the event the GUI becomes unresponsive and the user has no access to the console # stopping all threads # PackageManager actual initialziation # is done by setting the package count in dbus /Settings to 0 # remove dbus com.victronenergy.packageManager # uninstalling SetupHelper if requested above # done by trigging a nohup backround task that sleeps for 30 seconds then installs the package # reboot the system # exit # persistent storage for mainLoop packageIndex = 0 noActionCount = 0 lastDownloadMode = AUTO_DOWNLOADS_OFF bootInstall = False ignoreBootInstall = False DeferredGuiEditAcknowledgement = None lastTimeSync = 0 WaitForGitHubVersions = False # states for package.ActionNeeded REBOOT_NEEDED = 2 GUI_RESTART_NEEDED = 1 NONE = 0 def mainLoop (): global mainloop global PushAction global MediaScan global SystemReboot # initialized/used in main, set in mainloop, PushAction, InstallPackage global GuiRestart # initialized in main, set in PushAction, InstallPackage, used in mainloop global WaitForGitHubVersions # initialized above, set in UpdateGitHubVersion used in mainLoop global InitializePackageManager # initialized/used in main, set in PushAction, MediaScan run, used in mainloop global RestartPackageManager # initialized/used in main, set in PushAction, MediaScan run, used in mainloop global DeferredGuiEditAcknowledgement # set in the handleGuiEditAction thread becasue the dbus paramter can't be set there global noActionCount global packageIndex global noActionCount global lastDownloadMode global bootInstall global ignoreBootInstall global lastTimeSync startTime = time.time() # an unclean shutdown will not save the last known time of day # which is used during the next boot until ntp can sync time # so do it here every 30 seconds # an old RTC timeSyncCommand = '/etc/init.d/save-rtc.sh' if startTime > lastTimeSync + 30 and os.path.exists (timeSyncCommand): try: subprocess.Popen ( [ timeSyncCommand ], bufsize=-1, stdout=subprocess.PIPE, stderr=subprocess.PIPE) proc.commiunicate () # output ignored except: pass lastTimeSync = startTime packageName = "none" if DeferredGuiEditAcknowledgement != None: DbusIf.AcknowledgeGuiEditAction (DeferredGuiEditAcknowledgement) DeferredGuiEditAcknowledgement = None # auto uninstall triggered by AUTO_UNINSTALL_PACKAGES flag file on removable media # or SetupHelper uninstall was deferred # exit mainLoop and do uninstall in main, then reboot # skip all processing below ! actionMessage = "" bootReinstallFile="/etc/venus/REINSTALL_PACKAGES" currentDownloadMode = DbusIf.GetAutoDownloadMode () emptyPackageList = False checkPackages = True autoInstall = False autoDownload = False # hold off all package processing if package list is empty if len (PackageClass.PackageList) == 0: emptyPackageList = True checkPackages = False packageIndex = 0 # if boot-time reinstall has been requiested by reinstallMods # override modes and initiate auto install of all packages # ignore the boot reinstall flag if it's been done once and the flag removal failed elif os.path.exists (bootReinstallFile) and not ignoreBootInstall: # beginning of boot install - reset package index to insure a complete scan if not bootInstall: bootInstall = True packageIndex = 0 logging.info ("starting boot-time reinstall") elif WaitForGitHubVersions: checkPackages = False # don't look for new actions if uninstalling all packages or uninstalling SetupHelper elif MediaScan.AutoUninstall or SetupHelperUninstall or RestartPackageManager: pass # not doing something special - use dbus values else: autoDownload = currentDownloadMode != AUTO_DOWNLOADS_OFF autoInstall = DbusIf.GetAutoInstall () # download mode changed # restart at beginning of list and refresh all GitHub versions if currentDownloadMode != lastDownloadMode and currentDownloadMode != AUTO_DOWNLOADS_OFF: packageIndex = 0 checkPackages = False UpdateGitHubVersion.SetPriorityGitHubVersion ('REFRESH') # save mode so changes can be detected on next pass lastDownloadMode = currentDownloadMode # make sure a new scan starts at beginning of list if not checkPackages: packageIndex = 0 # process one package per pass of mainloop else: DbusIf.LOCK ("mainLoop 1") packageListLength = len (PackageClass.PackageList) # reached end of list - start over if packageIndex >= packageListLength: packageIndex = 0 # end of ONCE download - switch auto downloads off if currentDownloadMode == ONE_DOWNLOAD: DbusIf.SetAutoDownloadMode (AUTO_DOWNLOADS_OFF) currentDownloadMode = AUTO_DOWNLOADS_OFF # end of boot install if bootInstall: logging.info ("boot-time reinstall complete") bootInstall = False if os.path.exists (bootReinstallFile): try: os.remove (bootReinstallFile) except FileNotFoundError: pass except: # log the error and continue # set flag so we don't repeat the reinstall if the flag removal fails (until next boot) ignoreBootInstall = True logging.critical ("could not remove the boot time reinstall flag: /etc/venus/REINSTALL_PACKAGES") package = PackageClass.PackageList [packageIndex] packageName = package.PackageName packageIndex += 1 # skip conflict checks if boot-time checks are bening made package.UpdateVersionsAndFlags (doConflictChecks = not bootInstall) # disallow operations on this package if anything is pending packageOperationOk = not package.DownloadPending and not package.InstallPending if packageOperationOk and autoDownload and DownloadGitHub.DownloadVersionCheck (package): # don't allow install if download is needed - even if it has not started yet packageOperationOk = False actionMessage = "downloading " + packageName + " ..." PushAction ( command='download' + ':' + packageName, source='AUTO' ) # validate package for install if packageOperationOk and package.Incompatible == "" : installOk = False # one-time install flag file is set in package directory - install without further checks oneTimeInstallFile = "/data/" + packageName + "/ONE_TIME_INSTALL" if os.path.exists (oneTimeInstallFile): os.remove (oneTimeInstallFile) installOk = True # auto install OK (not manually uninstalled) and versions are different elif package.AutoInstallOk and package.PackageVersionNumber != package.InstalledVersionNumber: if autoInstall: installOk = True # do boot-time install only if the package is not installed elif bootInstall and package.InstalledVersion == "": installOk = True elif os.path.exists ("/data/" + packageName + "/AUTO_INSTALL"): installOk = True if installOk: packageOperationOk = False actionMessage = "installing " + packageName + " ..." PushAction ( command='install' + ':' + packageName, source='AUTO' ) DbusIf.UNLOCK ("mainLoop 1") # end if checkPackages DbusIf.LOCK ("mainLoop 2") actionsPending = False actionsNeeded = "" systemAction = NONE # hold off reboot or GUI restart if any package has an action pending # collect actions needed to activage changes - only sent to GUI - no action taken for package in PackageClass.PackageList: if package.DownloadPending or package.InstallPending: actionsPending = True # clear GitHub version if not refreshed in 10 minutes elif package.GitHubVersion != "" and package.lastGitHubRefresh > 0 and time.time () > package.lastGitHubRefresh + NORMAL_GITHUB_REFRESH + 10: package.SetGitHubVersion ("") if package.ActionNeeded == REBOOT_NEEDED: actionsNeeded += (package.PackageName + " requires REBOOT\n") systemAction = REBOOT_NEEDED elif package.ActionNeeded == GUI_RESTART_NEEDED: actionsNeeded += (package.PackageName + " requires GUI restart\n") if systemAction != REBOOT_NEEDED: systemAction = GUI_RESTART_NEEDED if systemAction == REBOOT_NEEDED: actionsNeeded += "REBOOT system ?" elif systemAction == GUI_RESTART_NEEDED: actionsNeeded += "restart GUI ?" # don't show an action needed if reboot, etc is pending if SystemReboot or GuiRestart or RestartPackageManager or InitializePackageManager: DbusIf.DbusService['/ActionNeeded'] = "" else: DbusIf.DbusService['/ActionNeeded'] = actionsNeeded DbusIf.UNLOCK ("mainLoop 2") if actionsPending: noActionCount = 0 else: noActionCount += 1 # wait for two complete passes with nothing happening # before checking to see if mainLoop should quit if noActionCount >= 2: if SystemReboot or InitializePackageManager or GuiRestart\ or RestartPackageManager or MediaScan.AutoUninstall or SetupHelperUninstall: # already exiting - include pending operations if systemAction == REBOOT_NEEDED: SystemReboot = True elif systemAction == GUI_RESTART_NEEDED: GuiRestart = True mainloop.quit() return False if actionMessage != "": DbusIf.UpdateStatus ( actionMessage, where='PmStatus' ) else: if emptyPackageList: idleMessage = "no active packages" elif bootInstall: idleMessage = "reinstalling packages after firmware update" elif WaitForGitHubVersions: idleMessage = "refreshing GitHub version information" elif currentDownloadMode != AUTO_DOWNLOADS_OFF and autoInstall: idleMessage = "checking for downloads and installs" elif currentDownloadMode == AUTO_DOWNLOADS_OFF and autoInstall: idleMessage = "checking for installs" elif currentDownloadMode != AUTO_DOWNLOADS_OFF and not autoInstall: idleMessage = "checking for downloads" else: idleMessage = "" DbusIf.UpdateStatus ( idleMessage, where='PmStatus' ) # enable the following lines to report execution time of main loop ####endTime = time.time() ####print ("main loop time %3.1f mS" % ( (endTime - startTime) * 1000 ), packageName) # to continue the main loop, must return True return True # end mainLoop # uninstall a package with a direct call to it's setup script # used to bypass package list, and other processing # # do not use once package list has been set up def directUninstall (packageName ): global SetupHelperUninstall global SystemReboot global GuiRestart if packageName == "SetupHelper": SetupHelperUninstall = True packageDir = "/data/" + packageName setupFile = packageDir + "/setup" try: if os.path.isdir (packageDir) and os.path.isfile (setupFile) \ and os.access(setupFile, os.X_OK): proc = subprocess.Popen ( [ setupFile, 'uninstall', 'runFromPm' ], bufsize=100000, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True ) proc.wait () # forward stdout and stderr lines from setup script to console/log file # note: all stdout lines will be output first, then all stderr lines # so won't be in cronological order ! for line in proc.stdout: logging.info ( line.strip () ) for line in proc.stderr: logging.error ( line.strip () ) returnCode = proc.returncode except: logging.critical ("could not uninstall " + packageName) else: if returnCode == EXIT_REBOOT: SystemReboot = True elif returnCode == EXIT_RESTART_GUI: GuiRestart = True # signal handler for TERM and CONT # this is needed to allow pending operations to finish before PackageManager exits # TERM sets RestartPackageManager which causes mainLoop to exit and therefore main to complete # TERM, then CONT is issued by supervise when shutting down the service # CONT handler differentiates a restart vs service down for logging purposes def setPmRestart (signal, frame): global RestartPackageManager RestartPackageManager = True def shutdownPmRestart (signal, frame): global RestartPackageManager global ShutdownPackageManager if RestartPackageManager: ShutdownPackageManager = True signal.signal (signal.SIGTERM, setPmRestart) signal.signal (signal.SIGCONT, shutdownPmRestart) # main # # ######## code begins here # responsible for initialization and starting main loop and threads # also deals with clean shutdown when main loop exits # def main(): global mainloop global SystemReboot # initialized/used in main, set in mainloop, PushAction, InstallPackage global GuiRestart # initialized in main, set in PushAction, InstallPackage, used in mainloop global InitializePackageManager # initialized in main, set in PushAction, used in mainloop global RestartPackageManager # initialized in main, set in PushAction, used in mainloop global ShutdownPackageManager global SetupHelperUninstall global WaitForGitHubVersions # initialized in main, set in UpdateGitHubVersion used in mainLoop SystemReboot = False GuiRestart = False InitializePackageManager = False RestartPackageManager = False ShutdownPackageManager = False SetupHelperUninstall = False # set logging level to include info level entries logging.basicConfig( format='%(levelname)s:%(message)s', level=logging.INFO ) # fetch installed version installedVersionFile = "/etc/venus/installedVersion-SetupHelper" try: versionFile = open (installedVersionFile, 'r') except: installedVersion = "" else: installedVersion = versionFile.readline().strip() versionFile.close() # if file is empty, an unknown version is installed if installedVersion == "": installedVersion = "unknown" logging.info ("PackageManager " + installedVersion + " starting") from dbus.mainloop.glib import DBusGMainLoop # Have a mainloop, so we can send/receive asynchronous calls to and from dbus DBusGMainLoop(set_as_default=True) # get platform global Platform platformFile = "/etc/venus/machine" try: file = open (platformFile, 'r') except: Platform = "???" else: machine = file.readline().strip() if machine == "einstein": Platform = "Cerbo GX" if machine == "cerbosgx": Platform = "Cerbo SGX" elif machine == "bealglebone": Platform = "Venus GX" elif machine == "ccgx": Platform = "CCGX" elif machine == "canvu500": Platform = "CanVu 500" elif machine == "nanopi": Platform = "Multi/Easy Solar GX" elif machine == "raspberrypi2": Platform = "Raspberry Pi 2/3" elif machine == "raspberrypi4": Platform = "Raspberry Pi 4" elif machine == "raspberrypi5": Platform = "Raspberry Pi 5" elif machine == "ekrano": Platform = "Ekrano GX" else: Platform = machine file.close() # initialze dbus Settings and com.victronenergy.packageManager global DbusIf DbusIf = DbusIfClass () # populate local package information from dbus settings # this creates package class instances PackageClass.AddPackagesFromDbus () global UpdateGitHubVersion UpdateGitHubVersion = UpdateGitHubVersionClass () global DownloadGitHub DownloadGitHub = DownloadGitHubPackagesClass () global InstallPackages InstallPackages = InstallPackagesClass () global AddRemove AddRemove = AddRemoveClass () global MediaScan MediaScan = MediaScanClass () # initialze package list # and refresh versions before starting threads # and the background loop DbusIf.ReadDefaultPackagelist () PackageClass.AddStoredPackages () # make a pass through the package list to: # update local versions and flags # ( this is needed anyway before mainLoop so do it here while we have the package ) # remove any packages if its name is not valid # (invalid package names may be left over from a previous version) # remove any packages with their forced removal flag is set # package conflicts are sometimes resolved by uninstalling a package # (done in their setup script eg GuiMods force removes GeneratorConnector) # remove duplicate packages # could be time-consuming (uninstall, removal and checking all packages) # lock is really unecessary since threads aren't running yet # # if a package is removed, start at the beginning of the list again while True: DbusIf.LOCK ("main") runAgain = False existingPackages = [] for (index, package) in enumerate (PackageClass.PackageList): packageName = package.PackageName # valid package name if PackageClass.PackageNameValid (packageName): flagFile = "/data/setupOptions/" + packageName + "/FORCE_REMOVE" # forced removal flag if os.path.exists (flagFile): os.remove (flagFile) if os.path.exists ("/etc/venus/installedVersion-" + packageName): logging.info ( "uninstalling " + packageName + " prior to forced remove" ) directUninstall (packageName) # now remove the package from list logging.warning ( "forced remove of " + packageName ) PackageClass.RemovePackage (packageIndex=index) runAgain = True break elif packageName in existingPackages: logging.warning ( "removing duplicate " + packageName ) PackageClass.RemovePackage (packageIndex=index, isDuplicate=True) runAgain = True break # invalid package name (including a null string) so remove the package from the list else: logging.warning ( "removing package with invalid name " + packageName ) PackageClass.RemovePackage (packageIndex=index) runAgain = True break # package not removed above - add its name to list that will be checked for duplicates existingPackages.append (packageName) DbusIf.UNLOCK ("main") if not runAgain: break del existingPackages DbusIf.UpdateDefaultPackages () #### start threads - must use LOCK / UNLOCK to protect access to DbusIf from here on UpdateGitHubVersion.start() DownloadGitHub.start() InstallPackages.start() AddRemove.start() MediaScan.start () # call the main loop - every 1 second # this section of code loops until mainloop quits GLib.timeout_add(1000, mainLoop) mainloop = GLib.MainLoop() mainloop.run() #### this section of code runs only after the mainloop quits (LOCK / UNLOCK no longer necessary) # output final prompts to GUI and log DbusIf.DbusService['/ActionNeeded'] = "" DbusIf.SetEditStatus ("") DbusIf.AcknowledgeGuiEditAction ('') message = "" if MediaScan.AutoUninstall: message = "UNINSTALLING ALL PACKAGES & REBOOTING ..." logging.warning (">>>> UNINSTALLING ALL PACKAGES & REBOOTING...") elif SetupHelperUninstall: message = "SetupHelper UNINSTALLED" logging.critical (">>>> SetupHelper UNINSTALLED") elif InitializePackageManager: if SystemReboot: message = "initializing and REBOOTING ..." logging.info (">>>> initializing PackageManager and REBOOTING SYSTEM") else: logging.info (">>>> initializing PackageManager ...") message = "initializing and restarting PackageManager ..." elif SystemReboot: message = "REBOOTING SYSTEM ..." logging.info (">>>> REBOOTING SYSTEM") elif GuiRestart: message = "restarting GUI and Package Manager..." elif ShutdownPackageManager: message = "shutting down PackageManager ..." elif RestartPackageManager: message = "restarting PackageManager ..." DbusIf.UpdateStatus ( message=message, where='PmStatus' ) DbusIf.UpdateStatus ( message=message, where='Editor' ) # stop threads, remove service from dbus logging.info ("stopping threads") UpdateGitHubVersion.StopThread () DownloadGitHub.StopThread () InstallPackages.StopThread () AddRemove.StopThread () MediaScan.StopThread () try: UpdateGitHubVersion.join (timeout=1.0) DownloadGitHub.join (timeout=1.0) InstallPackages.join (timeout=1.0) AddRemove.join (timeout=1.0) MediaScan.join (timeout=1.0) except: logging.critical ("one or more threads failed to exit") pass # if initializing PackageManager persistent storage, set PackageCount to 0 # which will cause the package list to be rebuilt from packages found in /data # user-specified Git Hub user and branch are lost if InitializePackageManager: DbusIf.DbusSettings['packageCount'] = 0 # com.victronenergy.packageManager no longer available after this call DbusIf.RemoveDbusService () # auto uninstall triggered by AUTO_UNINSTALL_PACKAGES flag file on removable media if MediaScan.AutoUninstall: # uninstall all packages EXCEPT SetupHelper which is done later for path in os.listdir ("/data"): directUninstall (path) SystemReboot = True if SystemReboot: try: # insure PackageManager service does not restart when reboot is needed proc = subprocess.Popen ( [ 'svc', '-o', '/service/PackageManager' ] ) # TODO: add -k for debugging - outputs message but doesn't reboot proc = subprocess.Popen ( "nohup sleep 5; shutdown -r now PackageManager is REBOOTING SYSTEM ... &", shell=True ) except: logging.critical ("system reboot command failed") elif GuiRestart: if os.path.exists ("/service/start-gui" ): command = [ 'svc', '-t', '/service/start-gui' ] else: command = [ 'svc', '-t', '/service/gui' ] try: proc = subprocess.Popen ( command ) except: logging.critical ("GUI restart failed") logging.info (">>>> PackageManager exiting") #### Initial entry point for program main() ================================================ FILE: ReadMe.md ================================================ # Overview The SetupHelper package provides: - a mechanism to automatically reinstall packages following a Venus OS update - an automatic update mechanism to keep packages up to date from GitHub archives or USB stick - control of the automatic download and install from the GUI - add and remove packages from the system - manual download, install, uninstall from the GUI - checks for package conflicts and prevents one package installing over another when the same files are modified - provides a "conflict resolution" option when such conflicts exist - a "blind" install of SetupHelper from SD/USB media - a blind uninstall mechanism which optionally includes reinstalling Venus OS - backup and restore SOME settings from `com.victronenergy.settings` This includes custom logos and copying logs to removable media - SetupHelper - PackageManager - gui - Restart or initialize PackageManager - Restart the GUI > [!NOTE] > Support for firmware prior to v3.10 has been dropped starting with SetupHelper v8.10 > if you are running older versions, change the branch/tag to preV3.10support > for any packages you wish to run on that firmware > > While this branch will remain active, there will be no features added to it > and only serious bug fixes will be applied. # Changes **SetupHelper v9** - changes to logging to comform to Victron rules (see below) **SetupHelper v8** - adds the ability for multiple packages to modify the same file - Packages must be written to "patch" a file rather than "replace" it **SetupHelper v7.0** - adds a conflict resolution mechanism. - Packages can identify known conflicts with other packages with a "packageDependencies" list - One package can specify that other packages must be uninstalled or installed before allowing the package to be installed - PackageManager also checks all files that will be modified to see if another package has already modified the same file. > [!NOTE] > All packages should be uninstalled, then reinstalled to create the necessary > information for these file-based conflicts to be identified. If a conflict exists it is reported on in the PackageManager menus and install is blocked. These conflicts can be resolved from within the Package editor menu. **SetupHelper v6.0** > [!NOTE] > SetupHelper v6.0 changes significantly from prevous versions - providing more automatic installation and installation of files, services and dBus Settings - v6.0 will install older packages but package setup scripts that utilize the new automated install and uninstall functions **will not work with SetupHelper v5.x** For this reason, packages that rely on the v6.0 setup helper functionality should also include a copy of the **HelperResources** found in SetupHelper v6.0 and newer Sourcing these helpers has also changed in v6.0. But there is also a backward compatible hook for older packages. The new sourcing mechanism can be found in the file `SetupHelper/HelperResources/forSetupScript`. # Logging Victron Energy has requested all packages refrain from writing to a log file in order to prevent runaway processes (e.g., services that crash and restart constantly) filling available space on /data. Therefore, when a setup script is run from the console, messages appear on the console but **are not logged** When the script is run from PackageManager, PackageManager forwards the script's messages to its log file. There are a few situations where a script will write directly to the PackageManager log file: - **reinstallMods** is called from rcS.local to reinstall PackageManager after a firmware update - **blind install** and **blind uninstall** # Helper resources Other packages use "helper resources" provided by SetupHelper Helper Resources simplify the package's setup script and include hooks that PackageManager uses to control installs and uninstalls. More information about Setup Helper and how to create packages that use it can be found in the file PackageDevelopmentGuidelines.md in the package directory. # Blind Install: By far, the easiest way to install SetupHelper is the "blind install" which requires no command-line interaction. 1. Download `venus-data-SetupHelperInstall.tgz` from the SetupHelper GitHub [repo](https://github.com/kwindrem/SetupHelper/raw/main/venus-data-SetupHelperInstall.tgz). > [!NOTE] > Mac OS and Safari are set by default to unzip packages. > The Open "safe" files after downloading (bottom of Safari Preferences General) > must be disabled in order to retain the zip file. 2. copy it to the root of a freshly formatted SD card or USB memory stick 3. place the media in the GX device (Cerbo, CCGX, etc) 4. reboot the GX device and allow the system to display the GUI - you should find the Package Manager menu at the bottom of the Settings menu 5. **REMOVE THE MEDIA** from the GX device after you see the GUI displayed Mechanisms are in place to prevent reinstallation, but removal is still a good idea! You should find the Package Manager menu at the bottom of the Settings menu Another way to install SetupHelper is to use the following from the command line of the GX device: ```bash wget -qO - https://github.com/kwindrem/SetupHelper/archive/latest.tar.gz | tar -xzf - -C /data rm -rf /data/SetupHelper mv /data/SetupHelper-latest /data/SetupHelper /data/SetupHelper/setup ``` You can also use the above procedure to install other packages. Simply substiture SetupHelper with the package name (e.g., GuiMods) in the above commands. However, using the PackageManager menus is by far easier. Once SetupHelper is installed, updates to it and other packages can be performed through the Classic UI using the PackageManager menus. > [!CAUTION] > Package Manager allows uninstalling SetupHelper. > > This can not be undone since the menus to control Package Manager will go away. You would need to use the Blind Install or run /data/SetupHelper/setup again to reinstall SetupHelper > > Note that removal does not actually remove the package so other setup scripts > will continue to function. > [!NOTE] > You can install other packages using wget as described above. > Or you can download the .tgz file and put that on a USB stick and plug that into the GX device. > > PackageManager will detect the file and install the package. # ssh access: Setting up ssh access with ssh keys is highly recommended for any system, but especially when installing third party extensions to Venus OS. Attaching a serial terminal for direct console access is another option, especially if you don't have a network setup. [This document](https://www.victronenergy.com/live/ccgx:root_access) describes ssh access and also serial terminal connections on Cerbo GX. Remote ssh access is now available via tailscale using the **TailscaleGX** package # System Recovery: It is unlikely, but some users have reported a package install leaving their system unresponsive or with a nonfunctional GUI (white screen). In this case, your options depend on the current state of the system. 1. (as always) reboot. This may clear the problem. 2. if you have a functioning GUI (either locally or via remote console, see if you can access the PackageManager menu. - If so, you can remove pacakges one at a time from there. - If you find an offeding package, post an issue to the GitHub repo for that package and include: - Platform (Cerbo, CCGX, Raspberry PI, etc) - Venus OS firmware version - Run a Settings backup and post the logs.zip file on the removable media. - Remove SetupHelper last since once you do, you loose the PackageManager menus! 3. if you have terminal or ssh access, try running the package setup scripts to uninstall packages one at a time. 4. try booting to the previous Venus OS version (in Stored backup firmware) Then perform a fresh Online firmware update to the latest version or use the .swu update via removable media. Use the Settings / Firmware / Stored backup firmware menu if you have GUI access. If you don't have GUI access, you can also switch to the backup version from the command line: ```bash /opt/victronenergy/swupdate-scripts/set-version.sh 2 ``` You can also force a firmware update from the command line if you have ssh or terminal access: - For on-line updates: ```bash /opt/victronenergy/swupdate-scripts/check-swupdate.sh -force -update ``` - For updates from removable media: ```bash /opt/victronenergy/swupdate-scripts/check-swupdate.sh -force -update -offline ``` 5. If PackageManager is still running, it will detect a file named AUTO_UNINSTALL_PACKAGES on removable media. - Create a file of that name (no extension, content unimportant) on a USB memory stick or SD card and insert this into the GX device. - The system should eventually reboot. In most cases, this should occur within 1-2 minutes. - After reboot, the system should come up in the stock configuration with no packages installed. - If the system does not reboot, it is likely PackageManager is no longer running, so try other options. - Remember to remove the media containing the `AUTO_UNINSTALL_PACKAGES` file to this will be repeated the next time PackageManager runs. 6. perform the Blind uninstall procedure below. **Finally:** - If you are running on a Raspberry PI, you can reimage the system SD card. - If you have a Cerbo, you can reimage it using this procedure: https://community.victronenergy.com/questions/204255/cerbo-gx-bricked-how-to-recover.html > [!NOTE] > This will wipe out all settings and you'll need to reconfigure the GX device from scratch. - The Victron "restore factory default" procedure can be used to will wipe out all settings. - You'll need to reconfigure the GX device from scratch. - However, it will NOT replace the operating system and Victron application, nor will it uninstall any packages. - You will most likely be locked out of ssh access since log-in information and ssh keys are stored in the /data partition which is completey erased by this procedure. - For this reason, I do not recommend using this as part of your attempt to recover a system with no GUI. # Blind UNINSTALL: A blind uninstall mechanism is provided to recover a system with an unresponsive GUI (white screen) or no ssh/terminal access. This will run all package setup scripts to uninstall that package from system files. In addition to uninstalling all packages, the blind uninstall can optionally reinstall VenusOS. To do so, include a `.swu` file for the platform and desired firmware version on the SAME removable media The archive for this is named `venus-data.UninstallAllPackages.tar.gz`. 1. Copy `venus-data.UninstallAllPackages.tar.gz` to a USB memory stick or SD card 2. Plug the removable media into the GX device 3. Reboot, wait 2 minutes and reboot a second time 4. When the system automatically reboots after the second manual one, remove the media. You should eventually see the GUI on the local display if there is one or be able to connect via remote console. > [!CAUTION] > Removing media or power cycling the GX device during the uninstall, > especially if reinstalling firmware could render the system unresponsive! > Wait to see the GUI before removing media or power cycling. Note that a firmware update can take several minutes to complete but will eventually reboot. When the blind uninstall finishes, `venus-data.UninstallAllPackages.tar.gz` file on the removable media is renamed (adding .XXX) so that the blind install will run only once. This renaming is necessary to prevent a loop where the system uninstalls and reboots. # System automatic configuration and package installation: It is possible to use SetupHelper to set up a new system based on a template saved from a working system. - Setup the working system the way you want the new system to behave including custom icons, - then perform a Settings backup. - Remove the flash drive from the GX device and plug into a computer that has internet access. - Copy `venus-data.tgz` from the SetupHelper GitHub repo to the same flash drive. - If you wish packages to also be installed, copy the package -latest.tgz file from those repos as well. - Create `SETTINGS_AUTO_RESTORE` on the flash drive (contents don't matter - file may be empty). - Create `AUTO_INSTALL_PACKAGES` on the flash drive as well. - Place the flash drive into the GX device to be configured and reboot (once for v2.90 or twice for prior versions). - **REMOVE THE FLASH DRIVE** after you have verified that all packages have been installed (check Active packages in PackageManager). ================================================ FILE: blindInstall/SetupHelperVersion ================================================ v9.4 ================================================ FILE: blindInstall/blindInstall.sh ================================================ #!/bin/bash # this script is part of a "blind install" archive which installs SetupHelper # # Simply inserting media into the GX device and rebooting once will install SetupHelper # # the process makes use of the Venus OS update-data.sh script run during system boot # archives named "venus-data.tgz" are unpacked during boot # overriting matching content in /data # # this archive unpacks to: # /data/SetupHelper-blind to avoid overwriting an existing copy of SetupHelper # /data/rc for the pre/post scripts # if versions of /data/SetupHelper-blind and the installed version of SetupHelper # DIFFER, OR if SetupHelper is NOT INSTALLED, # SetupHelper-blind replaces SetupHelper and the setup script is run # # pre-hook.sh and post-hook.sh scripts are run before and after the archive is unpacked # /data/rcS.local is saved in pre-hook.sh and restored in post-hook.sh. # The /data/rcS.local file included in the archive is never executed # In stead, post-hook.sh performs the version checks and calls blindInstall.sh # if appropriate. This eliminates the second reboot ! # In order to check versions prior to unpacking the archive, # the SetupHelper version is duplicated in the rc folder which unpacks to /data # BEFORE the SetupHelper-blind is unpacked. # # blindInstall.sh is run in the background so it can wait for dbus Settings resources # to become available before running the package install script. # source "/data/SetupHelper-blind/HelperResources/EssentialResources" logToConsole=false logMessage "starting" # wait until dbus settings are active while [ $(dbus -y | grep -c "com.victronenergy.settings") == 0 ]; do logMessage "waiting for dBus settings" sleep 1 done sleep 2 setupHelperBlind='/data/SetupHelper-blind' setupHelperStored='/data/SetupHelper' # move the extracted archive into position and run the setup script if [ -e "$setupHelperBlind" ]; then if [ -e "$setupHelperStored" ]; then logMessage "removing previous SetupHelper" rm -rf "$setupHelperStored" fi logMessage "moving SetupHelper (from blind archive) into position" mv "$setupHelperBlind" "$setupHelperStored" else logMessage "SetupHelper archive not found - no changes to package" fi # run the setup script if [ -f "$setupHelperStored/setup" ]; then logMessage "installing SetupHelper" "$setupHelperStored/setup" install auto else logMessage "error - can't install SetupHelper" fi # remove the blind install SetupHelper from the archive if still present rm -rf "$setupHelperBlind" logMessage "completed" ================================================ FILE: blindInstall/post-hook.sh ================================================ #!/bin/bash # this script is part of a "blind install" archive which installs SetupHelper # # refer to blindInstall.sh for more an explaination # logDir="/var/log/PackageManager" logFile="$logDir/current" if ! [ "$logDir" ]; then mkdir -P "$logDir" fi logMessage () { echo "$*" echo "blind install post-hook.sh: $*" | tai64n >> "$logFile" } logMessage "starting" # run the blind install script from the SetupHelper-blind script="/data/SetupHelper-blind/blindInstall/blindInstall.sh" if [ -f "$script" ]; then logMessage "running blindInstall.sh as background process" nohup "$script" > /dev/null & fi logMessage "completed" ================================================ FILE: blindInstall/pre-hook.sh ================================================ #!/bin/bash # this script is part of a "blind install" archive which installs SetupHelper # refer to blindInstall.sh for more an explaination # logDir="/var/log/PackageManager" logFile="$logDir/current" if ! [ "$logDir" ]; then mkdir -P "$logDir" fi logMessage () { echo "$*" echo "blind install pre-hook.sh: $*" | tai64n >> "$logFile" } logMessage "starting" scriptDir="$( cd "$(dirname $0)" >/dev/null 2>&1 ; /bin/pwd -P )" blindVersionFile="$scriptDir/SetupHelperVersion" installedVersionFile='/etc/venus/installedVersion-SetupHelper' setupHelperStored='/data/SetupHelper' # remove GitHub project data just in case it ends up on the target # (it's large (about 20 MB) and could get in the way of package replacement rm -rf $setupHelperStored/.git doInstall=false # SetupHelper is currently stored in /data # check to see if it needs to be updated if [ -d "$setupHelperStored" ]; then if [ -f "$blindVersionFile" ]; then blindVersion=$(cat "$blindVersionFile") else logMessage "ERROR: no blind version" blindVersion="" fi if [ -f "$installedVersionFile" ]; then installedVersion=$(cat "$installedVersionFile") else installedVersion="" fi if [ "$installedVersion" != "$blindVersion" ]; then doInstall=true fi # no SetupHelper found, skip version checks and install else doInstall=true fi # returning with 0 will trigger unpacking and run post-hook.sh if $doInstall ; then logMessage "completed - will do install" exit 0 # returning non-zero will prevent unpacking # there won't be an archive to unpack andpost-hook.sh will NOT run else logMessage "completed - unstall not needed - skipping unpack and install" exit -1 fi ================================================ FILE: blindInstall/rcS.localForUninstall ================================================ #!/bin/bash # this script is part of a "blind UNINSTALL" archive which UNINSTALLS all packages # Packages are not removed but marked so PackageManager does not auto intall later # # Venus OS will also be reinstalled if a suitable .swu file is found on removable media # log activity logDir="/var/log/PackageManager" logFile="$logDir/current" if ! [ "$logDir" ]; then mkdir -P "$logDir" fi logMessage () { echo "blindUninstall: $*" echo "blindUninstall: $*" | tai64n >> "$logFile" } logMessage "--- starting blind uninstall" # check to see if Venus OS will be reinstalled - actuall reinstall will be done later swCheckOutput=$(/opt/victronenergy/swupdate-scripts/check-updates.sh -offline -force -check) if (( $? == 0 )); then reinstallVenusOs=true swUpdateVersion=$(echo $swCheckOutput | awk '{print $NF}') else reinstallVenusOs=false swUpdateVersion="none" fi packages=$(ls -d /data/*) # run the script to anything that looks like a package (version and setup files) for package in $packages; do if [ -f "$package/version" ] && [ -f "$package/setup" ]; then packageName=$(basename $package) logMessage "uninstalling $packageName" "$package/setup" uninstall deferReboot deferGuiRestart auto fi done # remove all installed info in case some were missed in loop above rm -f /etc/venus/installedVersion* rm -rf "/etc/venus/installedModifications" # rename archive on removable media to prevent blindInstall from running again drives=$(ls /run/media/) for drive in $drives ; do archive="/run/media/$drive/venus-data-UninstallAllPackages.tgz" if [ -f "$archive" ]; then logMessage "renaming venus-data-UninstallAllPackages.tgz so blindUninstall won't run again" mv "$archive" "$archive.XXX" fi done # reinstall Venus OS - done in background so this script can clean up and exit without disrupting the software update if $reinstallVenusOs ; then logMessage "reinstalling Venus OS $swUpdateVersion" nohup sh -c 'sleep 1; /opt/victronenergy/swupdate-scripts/check-updates.sh -offline -force -update' > /dev/null & # reboot if not reinstalling Venus OS else logMessage "rebooting ..." nohup sh -c 'sleep 1; reboot' > /dev/null & fi # don't run this script again ! rm -f /data/rcS.local logMessage "--- ending blind uninstall" ================================================ FILE: changes ================================================ v9.4: added support for Raspberry PI 5 platform v9.3: fixed: patch error on RPI 5 v9.2: fixed: install failure during manual install does not force an uninstall add support for v3.70~71 v9.1: fixed: can't access package editor menu on firmware prior to v3.6 (regression in v9.0) v9.0: rewrite to insure logging is all done via multilog therefore, NO LOGGING when setup scripts are run from command line !!!!!!! updated blind install/uninstall removed support for firmware prior to v3.00 in various places v8.34 fixed: PackageManager Edit menu: buttons on bottom row sometimes missing fixed: syntax warning nv firmware 3.60 - warning only, everything working v8.33: fixed: boot-time reinstall flag removal failure caused PackageManager to crash if the flag could not be removed this would only happen if the file or partition did not have write permission v8.32: fixed: packages reinstalled at boot time with Auto Install off should only install if not currently installed and not manually uninstalled v8.31: fixed: installedFileList not being created which may leave modified files after uninstall fixed: .NO_ORIG files not removed when uninstalling v8.30: fixed: generator service not restarted (moved from dbus-generator-starter to dbus-generator) v8.29: fixed: active file not always updated by a patched file - SILENTLY ! v8.28: remove venus-os_ngrok from default package list first compatible version is now v3.10 v8.27: support the change to QtQuick 2 first used in v3.60~18 v8.26: added link to IncludeHelpers so that old packages can find correct file to source v8.25: changed ReadMe to a markup document included PackageDevelopmentGuidelines.md in the package (previously on DropBox) v8.24: 8.23 did not have the blind install files v8.23: add code to updatePackage and CommonResources to handle directory renames (e.g., dbus-generator-starter to dbus-generator) fixed: PackageManager crash if version string contains invalid characters updatePackage: fixed: USE_ORIGINAL flag files not created in some cases v8.22: fixed: missing log file and directories are not created until the PackageManager service starts so messages logged before that are not present in the log file v8.21: fixed: patch error for some files if package is reinstalled v8.20: add support for v3.50~22 (HTML style sheet in different location) add dbus-pi package to defaults list v8.19: fixed: system reboots after user chooses to reboot later v8.18: fixed: errors when installing RemoteGPIO v8.17: fixed: adding dbus setting didn't accommodate values starting in -- fixed: generator service was restarting unnecessarily when installing packages v8.16: fixed: no previous patch file error created in v8.15 v8.15: fixed: GUI restart doesn't always occur after install fixed: previouis patches file creation fails on first install because root fs was stil read only v8.14: fixed: crash when installing packages (introduced in v8.11) v8.12: readded blind install files v8.11: improved install/uninstall error handling in endScript () fixed: GUI and other service restarts not always happening v8.10: moved velib_python in SetupHelper to a single version dropping support for firmware earlier than v3.10 v8.9: fixed: further changes for the remote GUI issue v8.8: fixed: GuiMods web GUI (v1) broken (patched file permissions incorrect) v8.7: updatePackage: always rebuild patch files provide version-dependent velib_phthon for this and other packages v8.6: fixed: persistent download pending message after a download fails v8.5: fixed typo in version string (v8.4 was "v8.3=4") v8.4: Fixed: package install fails if it has older setup script (for a while, packages could have their own helper files) v8.3: added GitHub check frequency: 10 minutes, hourly, daily reduces network bandwidth v8.2: fixed: auto downloads can happen even when off (introduced in v8.1) v8.1: fixed: GitHub version refresses occuring too fast (~10 seconds vs 10 minutes) v8.0: allow multiple packages to modify the same file allow multiple patch files for each active file fixed: PackageManager hangs if there is no setup script in package directory fixed: "Once" download scan doesn't check all packages fixed: PackageManager hangs on Python 2 (Venus OS prior to v2.80) use HelperResources only from SetupHelper (not from package directory) this was necessary because /data/SetupHelper/patch is now used in place of the stock patch executable moved SetupHelper logging to /var/log/PackageManager/current from /var/log/SetupHelper added Recheck button for errors discovered in setup script prechecks PackageManager now completes pending operations before exiting add TailscaleGX to default package list updatePackage: added patch options including MANUAL to prevent automatic patch updates updatePackage: rewrite update file sets loop for speed improvement v7.18: fixed: only first service is uninstalled v7.17: services not always uninstalled v7.16: fixed: PackageManager hangs with package add v7.15: fixed: GitHub version not refreshed when user/branch change fixed: old blind install v7.14: fixed: incompatible message not cleared when package no longer incompatible v7.13: fixed: PackageManager doesn't install packages after firmware update v7.12: fixed: PackageManager hangs if there is no setup script in package directory v7.11: fixed: conflicts not cleared when they have been resolved v7.10: fixed: services with dash in the name do not install/uninstall v7.9: added blind install .tgz files v7.8: fixed: packages getting downloaded when not needed + PackageManger crash results in corrupted pacakges v7.7: fixed: remove duplicates a package in Active packages v7.6: fixed: Package editor menus shows Now / Later in stead of Proceed / Cancel for Show Details v7.5: fixed: PackageManager restarts when removing package v7.4: version bump -- HelperResources version was not updated in v7.3 v7.3: fixed: GitHub versions are not refreshed when reentering the Active Packages menu fixed: unrecognized command showDetails fixed: can't remove packages from Package editor menu improve reporting of restart/reboot requirements in GUI v7.2: more fixes for install failure GUI lockups fixed: repeating "checking" messages (and the assocated calls to setup scripts) v7.1: fixed: install failure locks up GUI check for and report patching errors v7.0: fixed: Package editor menu sometimes locks up with Download, etc grayed out fixed: file set error uninstalls package without prompting for action fixed: running setup script from inside the package directory fails e.g., cd /data/GuiMods; ./setup add package dependencies, conflict detection and resolution added file system check and status to PackageManager without installing the package so incomplete file set displayes on the GUI without an install attempt fixed: RemoteGPIO hangs during install add RemoteGPIO to default package list improved reporting of errors in Package editor most issues are now identified BEFORE an install report no file set and incomplete file set separately no file set allows install, incomplete does not v6.13: (betas only - never released) needed to skip to 6.13 so HelperFiles selection would always work v6.12: (betas only - never released) v6.11: skipped v6.10: enable auto download and install after package add previously, packagew would not auto install if uninstalled manually v6.9: fixed: file uninstall fails if extracted from setup script removed PackageManager download delays v6.8: fixed: PackageManager won't download if package directory doesn't exist bug created in or about v6.5 v6.7: fixed: GUI white screen for versions prior to v3.00 v6.6: fixed: PackageManager hangs on remove fixed: PackageManager not setting no file set for incomplete fs v6.5: fixed: package download failure not always reported on the GUI v6.4: fixed: setup scripts fail if no fileList file updatePacakge: fixed problems related to incompatible versions updatePackage: fixed: helper resources updated even if there were no changes v6.3: fixed: removing active pachage freezes PackageManager v6.2: fixed: repeaded PackageManager install attempts when failure occurs v6.1: fixed: white screen for VenusOs prior to v3.00 v6.0: PackageManager: clear no file set flag when stored version changes add automated install, uninstall based on file, services and dBuse Settings lists moved package reinstall after Venus OS update to PackageManager only SetupHelper installed by reinstallMods now the unix patch facility is available for modifying replacement files this may not work for version-dependenet files however "helper resources" can now reside in the package directory in addition to SetupHelper. The code checks for the newest copy and uses that v5.18 fixed: installs fail on CCGX (resize2fs failure) v5.17: fixed: root not always resized after firmware update v5.16: fixed: white/black screen on first boot after firmware update incorporate changes for GUI v1 and gui-v2 selection, mainly to prevent package install if GUI v1 is needed and missing v5.15: fixed: PackageManager isn't in menus after v5.14 install updateFileSets: fixed: NO_REPLACEMENT in existing file sets that should link to other sets v5.14: v5.13 did not include blind install v5.13: fixed: GUI not restarted in v3.20~26 fixed: COMPLETE flag not set when creating file new file set this isn't critical, just slows down installs because file set must be rechecked updateFileSets: fixed: USE_ORIGINAL not updated proerly fixed: typo in package manager backup/restore menu v5.12: add support for gui-v2 documents and screen shots moved to a public DropBox: https://www.dropbox.com/scl/fo/bx5aftvgrqq0vp060mwip/h?rlkey=k28c2i49fjfpcyjfsuldwp159&dl=0 v5.11: check for room on file systems before allowing install resize root partition before installing a package the above issues are critical to avoid bricking systems !!!!!! updateFileSets: better error checking and more status while running long loops a replacement file and USE_ORIGINAL flag was incorrectly allowed the replacement file has priority during package install so this was not a severe issue, but USE_ORIGINAL is now removed in this case v5.10: updateFileSets: check for errors before moving version-independent files v5.9: updateFileSets fixed: creating symlinks in new version sometimes fails v5.8: fixed bugs in updateFileSets introduced in v5.7 v5.7: updateFileSets: make changes to the copy of the package instead of to the main package directory so package is not updated until changes are accepted updateFileSets: check for version-independent files in file sets and move them from the file sets to FileSets/ version-independent files are those that do not havea stock file and have only one real file in all file sets (other file sets have links) v5.6: fixed: version number segments starting with 0 interperted as octal updateFileSets: added package backup / restore changed status messages to: no file set for vxx.yy~zz v5.5: fixed: status text in Active packages menu is black - should be white updateFileSets: fixed: reported no package errors when there were some v5.3 / v5.4: updateFileSets: fixed: stale symlinks not removed updateFileSets: remove existing file sets that only contain sym links and are not in the stock version list updateFileSets: flag file sets that only contain sym links this aids managing file sets updateFileSets: add progress indication to differentiate from a hung app checks can now take significant amount of time v5.1/5.2: removed original file symbolic links they are not needed (even for older verisons of SetupHelper) v5.0: _checkFileSets in CommonResources: if file set exists and contains the COMPLETE flag file, skip all checks otherwise, proceed with file set validation or attempt to create one for the current Venus OS version updateFileSets now fills in all file sets with symlinks so that the install does not have to search for a matching original the search for a match has reportedly failed in a few cases for unknown reasons v4.43: fixed: PackageManager crashes when reinitializing database fixed stuck Package Manager status messages v4.42: fixed: updated services are not always restarted add ExtTransferSwitch to default package list v4.41: added support for dark mode (thanks mr-manual) v4.40: fixed bug in updateFileSets that resulted in no replacement file errors an incorrect not a released version warning v4.39: updateFileSets now allows creating file sets for only released versions installService in ServiceResources now supports multiple services in package fixed: install button in package edit menu sometimes drawn in wrong posisition v4.38: fixed: v4.37 overwrote settingsOptions and added logs.zip to /dataw v4.37: add settings backup to local storage (/data/settingsBackup) v4.36: fixed: crash if version file contains a zero length string (not counting white space) split add stored packages processing to reduce execution time in main loop enhanced Git Hub version updates: longer background updtate time (10 minutes vs 1 minute) complete refresh when entering Active packages menu v4.35: fixed: package edit menu buttons are not active after v4.34 update v4.34: disable GitHub version updates if automatic downloads are off eliminates internet traffic needed to retrieve version remove ._ files from blind install archives added BatteryAggregator (pulquero) to default packages updateFileSets now place files at highest possible version makes searches faster and makes removing old file sets easier checkFileSets now searches version list from highest version to lowest to improve file set creation speed v4.33: fixed: menu items not being hidden added all logs to Settings backup added optional Venus OS firmware update to blindUninstall v4.32: add some packages to the default list v4.31: fixed: PackageManager edit menus not working v4.30: fixed: can't select many items in PackageManager menus add Cerbo SGX platform add ExtTransferSwitch to default package list add FroniusSmartmeter to default package list add dbus-i2c to default package list add DCSystemAggregator to default package list add gatt-exec-server to default package list v4.28/v4.29: make SetupHelper independent of Venus OS version v4.27: fixed 20 MB size for blind isntall archive v4.25: fixed: downloads triggered from the PackageManager edit menu are sometimes are delayed by automatic download checks fixed: fast download sometimes ends before all packages are checked removed Fast, then Normal download mode fast downloads are now automatic when enabling downloads v4.24: fixed: incompatible version check somtimes fails v4.23: added Cerbo tanks and temps backup/restore v4.22: fixed: reinstallMods does not reboot or restart the GUI if needed included detailed description / help creating setup scripts and file sets v4.21: add missing settings to backup/restore v4.20: added support for v2.90~22 v4.19: fixed: after a blind install rcS.local did not get updated optimized reinstallMods - wait for dbus only if script needs to run v4.18: check versions before installing a package from removable media check versions before transferring a package from removable media these prevent a package with ONE_TIME_INSTALL set from installing over and over agin if the removable media is left in place and the system rebooted additional fixes for reinstall not working after an OS update v4.17: fixed: reinstall not working after OS update added AUTO_INSTALL_PACKAGES flag to /data this flag is easier to build into an archive than the one on removable media but is removed following the auto install to prevent repeats added AUTO_INSTALL flag in each package this overides the user auto install preferece dropped support for Venus OS v2.4x and 2.5x v4.16: fixed: white screen in Venus v2.73 and earlier v4.15: released - no changes v4.15~7: added delays in install service so things get initialized properly v4.15~6: added blind UNINSTALL via a special venus-data.tar.gz file see instructions in the ReadMe v4.14~5: added PackageManager persistent storage initialize Both the INITIALZE_PACKAGE_MANAGER flag file on removable media and a menu item has been added that will trigger the PackageManager dbus Setting storageto be initialized, then PackageManager restarted. The storage is then rebuilt when PackageManager starts back up. added UNNSTALL_ALL_PACKAGES removable media flag If this file is found on removable media, PackageManager will UNINSTALL ALL packages including SetupHelper these additions help recover systems without a user interface to factory conditions, including a blank or unresponsive GUI v4.14~4: updated ReadMe fixed: auto eject occured on manual settings restore should be just AUTOMATIC restores v4.14~3: add auto eject fixed: couldn't backup or restore settings v4.14~2: settings restore now creates missing parameters rewrote blind install to use the pre/post hooks for v2.90 blind install still works with prior Venus OS versions added AUTO_INSTALL_PACKAGES flag file on removable media functions same as enabling auto install in PackageManager menu added support for new /service mechanisms in v2.90 v4.14~1: add settings auto restore if SETTINGS_AUTO_RESTORE flag file exists on removable media v4.13: add logs as part of settings backup v4.12: added checks for file set errors before attempting auto install v4.11: added support for Venus OS v2.90~3 firmware v4.9/10: added dbus Settings cleanup to remove invalid packages v4.8: fixed: blind install fails if stored SetupHelper version is newer than archive version but SetupHelper not currently installed. v4.7: fixed: another bug matching original files from released and large verisons v4.6: fixed: released version file set skipped for versions still in beta this typically only happens for a large version after the small version is released v4.5: forgot to update blind install files v4.4: added v2.80 - no functional changes v4.3: include package setup script options in settings backup/restore v4.2: fixed: intermittent crash on initialization fixed: PackageManager doesn't always start after installing SetupHelper with the service overlay when uninstalling then installing SetupHelper, PackageManager didn't start fixed: field reports of package with no name in active package list remove any such packages during initialization v4.1: fixed: packageManager crash when moving old DO_NOT ... flags to setupOptions fixed: white screen on v2.8~33-large-24 v4.0: beta test period ended added running version to PackageManager sign-on v4.0~38: fixed: backup/restore hangs v4.0~37: changed the blind install process to minimize issues if venus-data.tgz is left mounted SetupHelper now unpacks to /data/SetupHelper-blind, then is moved to /data/SetupHelper and the setup script run ONLY IF it is a newer version v4.0~36: blind install was't updated for ~35 !!! v4.0~35: reinstall now compares installed and package versions and installs if they differ previously, booting to the alternate installed Venus version would not trigger a package reinstall, possibly resulting in problems or at least out of date packages v4.0~34: added image overlays to backup/restore this includes custom logos for Mobile and Tile overviews v4.0~33: fixed: PackageManager doesn't run on Venus versions prior to v2.80~10 (Python 2). v4.0~32: fixed: version numbers not in the Victron format would crash PackageManager also accommodate a other version string formats: vX.Y.Z, vX.YdZ, vX.YaZ, vX.YbZ v4.0~31: fixed crashes and bugs that prevented initial install on a system that has no packages yet v4.0~30: added settings backup/restore note this is NOT the Victron mechanism rather, it extracts SOME of the Settings parameters and writes these to a file care was taken to save/restore only those parameters that should not cause conflicts When Victron releases their mechanism, this one will be removed v4.0~26: fixed: Large features not appearing fixed: typo in 123SmartBMS-Venus in defaultPackageList v4.0~25: handle nonexistant package directory pull GitHub user/branch from package directory file optimize AddStoredPackages so it can run all the time (some updates were being missed) changed default package name: smartbms-venus to 123SmartBMS-Venus v4.0~24: add Reboot/GUI restart button to Package Manager main menu to address deferred operations ("Later") v4.0~23: fixed: GUI restart not happening after auto install fixed: GUI restart notificaiton in menu not cleared after GUI restart v4.0~22: not used v4.0~21: fixed: adding package didn't carry over GitHub user and branch v4.0~20: rearranged package editor menus Package version list -> Active packages tapping on an entry leads to Package editor added Inactive packages which shows only packages that are not on the system yet (or manually removed) tapping on entry leads to Add package menu added separate Add package menu Package Editor eliminated from main menu (access through Active packages only) default packages are no longer automatically added to the active package list v4.0~19: fixed: venus-data.tar.gz in v2.80~18 didn't include PackageManager.py v4.0~18: fixed: packages auto add/install when PackageManager is restarted even if REMOVED / DO_NOT_AUTO_INSTALL was set more work on GUI getting "stuck" bogus "unpack tar from GitHub failed" message - package downloads properly major change to thread structure to make operations more responsive and to minimize CPU consumption when idle v4.0~17: fixed: package editor status did not always show package name fixed?: occationally, Package Editor appears to get stuck when a PackageManager action completes. A missed property update from the dbus paramter may be yhe cause Set a timer to refresh properties in the GUI v4.0~16: fixed: packages auto auto-adding following manual removal v4.0~15: fixed: auto download not working moved dbus settings for PackageManager to /Settings/PackageManager from /Settings/PackageMonitor remove dbusSettings when package is removed previously these were left in place changed menu items and titles to conform to Victron standards: only firt word capitalized v4.0~14: fixed: selecting "Now" in GUI when reboot needed does nothing fixed: blind install did not work v4.0~13: added support for Venus OS v2.80~33-large-24 v4.0~12: fixed: manually uninstalled packages would reinstall immediately if auto install was on accommodate Python 2.7 for Venus OS prior to v2.80~10 fixed: a setup script run failure was not handled properly and caused the install thread to hang v4.0~11: fixed bug that caused GUI to restart repeatedly if package was not compatible with the current Venus version e.g., a file set error major rewrite to PackageManager download code changes to GUI: Add Package -> New Package moved SetpHelper uninstall warning to status moved action confirmaiton message to status, Confirm ... button now reads Proceed removed "can't remove" ... message v4.0~10: GitHub downloads and SD/USB transfers now scan the entire directory tree searching for a package directory. This was done because of the 123 smartBMS archive directory structure but there are other issues preventing integration with PackageManger. Fixed bug that showed a blank status line with the OK button after a download Rhe OK button no longer appears and Package Editor menu returns to the "navigation" mode v4.0~9: fixed bug with firstCompatibleVersion added try: / except: around all subprocess.run calls so if the call fails, the program continues to run v4.0~8: refresh GitHub version prior to download checks new upadates to GitHub could be missed reduced GitHub vesion refres delay touching a row in Package Version List menu leads to Package Editor menu and < will return to the version list v4.0~5 - 7: download bug fixes v4.0~4: add Package Manager & GUI add setup script return codes for above add optionsRequired flag file (VeCanSetup is only package that needs this now) add platform and version checks to CommonResources add install opiton to CommonResources better support installs without command line ### TBD remove logging to package log files improve adding packages from SD/USB split auto download and auto install ================================================ FILE: defaultPackageList ================================================ # the DEFAULT list of packages managed by SetupHelper # actual list is based on what is stored on the system # this list assists in adding new packages # lines beginning with # are ignored and can be used # to remove a package from auto and manual updates # or as comments # blank lines are ignored # incomplete lines are ignored # Package GitHubUser Tag/branch/version SetupHelper kwindrem latest GuiMods kwindrem latest ShutdownMonitor kwindrem latest VeCanSetup kwindrem latest RpiDisplaySetup kwindrem latest RpiGpioSetup kwindrem latest TailscaleGX kwindrem latest 123SmartBMS-Venus 123electric latest RpiTemperature TimD1981 latest BatteryAggregator pulquero latest dbus-i2c pulquero latest DCSystemAggregator pulquero latest gatt-exec-server pulquero latest dbus-pi pulquero latest FroniusSmartmeter SirUli main RemoteGPIO Lucifer06 main ================================================ FILE: forSetupScript ================================================ #### add the following lines to the package's setup script #### following line incorporates helper resources into this script source "/data/SetupHelper/HelperResources/IncludeHelpers" #### end of lines to include helper resources ================================================ FILE: genericSetupScript ================================================ #!/bin/bash # this script will install any package that can use # the automated install and uninstall mechanisms provided by SetupHelper # that is, no custom prompting for command line exection # and no custom installation such as editing replacement files # # link the package's setup script to this one: # ln -s /data/SetupHelper/genericSetupScript /data//setup # tell CommonResources to: # prompt for install/uninstall # auto install or auto uninstall # then exit # CommonResources will NOT return here ! standardPromptAndActions='yes' #### following line incorporates helper resources into this script source "/data/SetupHelper/HelperResources/IncludeHelpers" #### end of lines to include helper resources # never returns from CommonResources ! ================================================ FILE: gitHubInfo ================================================ kwindrem:latest ================================================ FILE: makeVelib_python ================================================ #!/bin/bash # convert a version string to an integer to make comparisions easier # # Note: copied from VersionResources # but also includes code to report duplcates not in the VersionResources version function versionStringToNumber () { local version="$*" local numberParts local versionParts local numberParts local otherParts local other local number=0 local type='release' # split incoming string into # an array of numbers: major, minor, prerelease, etc # and an array of other substrings # the other array is searched for releasy type strings and the related offest added to the version number read -a numberParts <<< $(echo $version | tr -cs '0-9' ' ') numberPartsLength=${#numberParts[@]} if (( $numberPartsLength == 0 )); then versionNumber=0 versionStringToNumberStatus="$version: invalid, missing major version" return 1 fi if (( $numberPartsLength >= 2 )); then read -a otherParts <<< $(echo $version | tr -s '0-9' ' ') for other in ${otherParts[@]}; do case $other in 'b' | '~') type='beta' (( number += 60000 )) break ;; 'a') type='alpha' (( number += 30000 )) break ;; 'd') type='develop' break ;; esac done fi # if release all parts contribute to the main version number # and offset is greater than all prerelease versions if [ "$type" == "release" ] ; then (( number += 90000 )) # if pre-release, last part will be the pre release part # and others part will be part the main version number else (( numberPartsLength-- )) (( number += 10#${numberParts[$numberPartsLength]} )) fi # include core version number (( number += 10#${numberParts[0]} * 10000000000000 )) if (( numberPartsLength >= 2)); then (( number += 10#${numberParts[1]} * 1000000000 )) fi if (( numberPartsLength >= 3)); then (( number += 10#${numberParts[2]} * 100000 )) fi versionNumber=$number versionStringToNumberStatus="$version:$number $type" return 0 } totalErrors=0 totalWarnings=0 packageErrors=0 packageWarnings=0 outputtingProgress=false function logMessage () { if $outputtingProgress ; then clearProgress fi echo "$*" if [[ "$*" == "ERROR"* ]]; then ((totalErrors++)) ((packageErrors++)) elif [[ "$*" == "WARNING"* ]]; then ((totalWarnings++)) ((packageWarnings++)) fi } function outputProgressTick () { if ! $outputtingProgress ; then echo -en "$beginProgressString" fi echo -en "$1" outputtingProgress=true } function clearProgress () { # start a new line if outputting ticks if $outputtingProgress; then echo # echo -ne "\r\033[2K" #### erase line fi outputtingProgress=false } beginProgressString="" function beginProgress () { # erase the line but stay on it if $outputtingProgress ; then clearProgress fi if [ ! -z "$1" ]; then beginProgressString="$1 " echo -en "$beginProgressString" outputtingProgress=true fi } #### script code begins here # attempt to locate SharedUtilities based on the location of this script # (it is assumed to be in the SetupHelper directory) # also sets the package root directory based on this also # and also the stock files base directory # # if these are not correct, edit the lines below to set the appropriate values scriptDir="$( cd $(dirname "$0") >/dev/null 2>&1 ; /bin/pwd -P )" packageRoot="$( dirname $scriptDir )" stockFiles="$packageRoot/StockVenusOsFiles" pythonLibDir="opt/victronenergy/dbus-systemcalc-py/ext/velib_python" veLibFiles=( vedbus.py dbusmonitor.py settingsdevice.py ve_utils.py ) #### set these as appropriate to your system if the values set above are not correct #### packageRoot=FILL_THIS_IN_AND_UNCOMMENT_LINE #### stockFiles=FILL_THIS_IN_AND_UNCOMMENT_LINE if [ ! -e "$packageRoot" ]; then echo "unable to locate package root - can't continue" exit elif [ ! -e "$stockFiles" ]; then echo "unable to locate stock files - can't continue" exit fi # make the version list from the directories in stock files # version lists are sorted so the most recent version is first tempList=() stockVersionList=($(ls -d "$stockFiles"/v[0-9]* 2> /dev/null)) for entry in ${stockVersionList[@]} ; do version=$(basename $entry) versionFile="$stockFiles/$version/opt/victronenergy/version" if [ -f "$versionFile" ]; then realVersion=$(cat "$versionFile" | head -n 1) else logMessage "ERROR version file missing from stock files $version - can't continue" exit fi if [ $version != $realVersion ]; then logMessage "ERROR $version name does not mactch Venus $realVersion - can't continue" exit fi if versionStringToNumber $version ; then tempList+=("$version:$versionNumber") else logMessage "ERROR invalid version $versionStringToNumberStatus - not added to list" fi done stockVersionList=( $(echo ${tempList[@]} | tr ' ' '\n' | sort -t ':' -r -n -k 2 | uniq ) ) stockVersionListLength=${#stockVersionList[@]} if (( stockVersionListLength < 2 )); then logMessage "fewer than 2 versions - nothing to compare" exit fi if [ -e "$scriptDir/velib_python" ]; then rm -rf "$scriptDir/velib_python" fi mkdir -p "$scriptDir/velib_python" for (( i1 = 0; i1 < $stockVersionListLength; i1++ )); do newVersion=false IFS=':' read version versionNumber <<< "${stockVersionList[$i1]}" if (( i1 == 0 )); then newVersion=true else for file in ${veLibFiles[@]} ; do file1="$stockFiles/$version/$pythonLibDir/$file" file2="$stockFiles/$previousVersion/$pythonLibDir/$file" if ! cmp -s "$file1" "$file2" > /dev/null ; then logMessage " $file $previousVersion $version differ" newVersion=true fi done fi if $newVersion ; then if (( i1 == 0 ));then velibDir="$scriptDir/velib_python/latest" prevVelibDir="$scriptDir/velib_python/latest" else velibDir="$scriptDir/velib_python/$version" fi mkdir "$velibDir" logMessage "new velib_python version $version" for file in ${veLibFiles[@]} ; do file1="$stockFiles/$version/$pythonLibDir/$file" file2="$velibDir/$file" cp -f "$file1" "$file2" done newVersion=false previousVersion=$version prevVelibDir="$velibDir" fi echo $version > "$prevVelibDir/oldestVersion" done ================================================ FILE: rcS.local ================================================ #!/bin/bash # SetupHelper reinstall if [ -f /data/SetupHelper/reinstallMods ]; then nohup /data/SetupHelper/reinstallMods > /dev/null & fi #end SetupHelper reinstall ================================================ FILE: reinstallMods ================================================ #!/bin/sh # this script is called from /data/rcS.local during system boot # it checks to see the PackageManager service is installed and if not, # will install ONLY the PackageManager service # # the REINSTALL_PACKAGES flag file is then set so that # when PackageManger runs, it will do boot-time reinstall checks for all packages # PackageManager then clears this flag when all install checks have been made scriptDir="$( cd "$(dirname $0)" >/dev/null 2>&1 ; /bin/pwd -P )" helperResourcesDir="$scriptDir/HelperResources" source "$helperResourcesDir/EssentialResources" source "$helperResourcesDir/ServiceResources" # disable outputting log messages to console logToConsole=false if [ -f "$setupOptionsDir/DO_NOT_AUTO_INSTALL" ]; then logMessage "CRITICAL: SetupHelper was manually uninstalled therefore it was not reinstalled" logMessage " other packages will NOT BE REINSTALLED either !" # remove lines from rcS.local that call reinstallMods so this doesn't happen repeatadly sed -i -e "/# SetupHelper reinstall/,/fi/d" "$rcLocal" # install PackageManager service else # installing the PackageManager service requires remounting root R/W updateRootToReadWrite if ! $installFailed ; then # install PackageManager service if not yet installed if ! [ -e "$serviceDir/PackageManager" ]; then logMessage "installing PackageManager service - PackageManager will reinstall all packages" installService PackageManager fi fi if ! $installFailed ; then touch "/etc/venus/REINSTALL_PACKAGES" else logMessage "reinstallMods not completed - packages will not be reinstalled" fi fi ================================================ FILE: services/PackageManager/log/run ================================================ #!/bin/sh exec multilog t s25000 n4 /var/log/PackageManager ================================================ FILE: services/PackageManager/run ================================================ #!/bin/sh exec 2>&1 exec /data/SetupHelper/PackageManager.py ================================================ FILE: settingsList ================================================ /Settings/Alarm/Audible /Settings/Alarm/System/GridLost /Settings/Alarm/Vebus/HighDcCurrent /Settings/Alarm/Vebus/HighDcRipple /Settings/Alarm/Vebus/HighDcVoltage /Settings/Alarm/Vebus/HighTemperature /Settings/Alarm/Vebus/InverterOverload /Settings/Alarm/Vebus/LowBattery /Settings/Alarm/Vebus/TemperatureSenseError /Settings/Alarm/Vebus/VeBusError /Settings/Alarm/Vebus/VoltageSenseError /Settings/Ble/Service/Pincode /Settings/CGwacs/AcPowerSetPoint /Settings/CGwacs/BatteryLife/DischargedTime /Settings/CGwacs/BatteryLife/Flags /Settings/CGwacs/BatteryLife/MinimumSocLimit /Settings/CGwacs/BatteryLife/Schedule/Charge/0/Day /Settings/CGwacs/BatteryLife/Schedule/Charge/0/Duration /Settings/CGwacs/BatteryLife/Schedule/Charge/0/Soc /Settings/CGwacs/BatteryLife/Schedule/Charge/0/Start /Settings/CGwacs/BatteryLife/Schedule/Charge/1/Day /Settings/CGwacs/BatteryLife/Schedule/Charge/1/Duration /Settings/CGwacs/BatteryLife/Schedule/Charge/1/Soc /Settings/CGwacs/BatteryLife/Schedule/Charge/1/Start /Settings/CGwacs/BatteryLife/Schedule/Charge/2/Day /Settings/CGwacs/BatteryLife/Schedule/Charge/2/Duration /Settings/CGwacs/BatteryLife/Schedule/Charge/2/Soc /Settings/CGwacs/BatteryLife/Schedule/Charge/2/Start /Settings/CGwacs/BatteryLife/Schedule/Charge/3/Day /Settings/CGwacs/BatteryLife/Schedule/Charge/3/Duration /Settings/CGwacs/BatteryLife/Schedule/Charge/3/Soc /Settings/CGwacs/BatteryLife/Schedule/Charge/3/Start /Settings/CGwacs/BatteryLife/Schedule/Charge/4/Day /Settings/CGwacs/BatteryLife/Schedule/Charge/4/Duration /Settings/CGwacs/BatteryLife/Schedule/Charge/4/Soc /Settings/CGwacs/BatteryLife/Schedule/Charge/4/Start /Settings/CGwacs/BatteryLife/SocLimit /Settings/CGwacs/BatteryLife/State /Settings/CGwacs/MaxChargePercentage /Settings/CGwacs/MaxChargePower /Settings/CGwacs/MaxDischargePercentage /Settings/CGwacs/MaxDischargePower /Settings/CGwacs/MaxFeedInPower /Settings/CGwacs/OvervoltageFeedIn /Settings/CGwacs/PreventFeedback /Settings/CGwacs/RunWithoutGridMeter /Settings/CanBms/SocketcanCan0/ProductId /Settings/Canbus/can0/Profile /Settings/Canbus/can1/Profile /Settings/Devices/vebus_ttyUSB0/CustomName /Settings/Devices/vebus_ttyUSB1/CustomName /Settings/DigitalInput/1/AlarmSetting /Settings/DigitalInput/1/Count /Settings/DigitalInput/1/CustomName /Settings/DigitalInput/1/InvertAlarm /Settings/DigitalInput/1/InvertTranslation /Settings/DigitalInput/1/Multiplier /Settings/DigitalInput/1/Type /Settings/DigitalInput/2/AlarmSetting /Settings/DigitalInput/2/Count /Settings/DigitalInput/2/CustomName /Settings/DigitalInput/2/InvertAlarm /Settings/DigitalInput/2/InvertTranslation /Settings/DigitalInput/2/Multiplier /Settings/DigitalInput/2/Type /Settings/DigitalInput/3/AlarmSetting /Settings/DigitalInput/3/Count /Settings/DigitalInput/3/CustomName /Settings/DigitalInput/3/InvertAlarm /Settings/DigitalInput/3/InvertTranslation /Settings/DigitalInput/3/Multiplier /Settings/DigitalInput/3/Type /Settings/DigitalInput/4/AlarmSetting /Settings/DigitalInput/4/Count /Settings/DigitalInput/4/CustomName /Settings/DigitalInput/4/InvertAlarm /Settings/DigitalInput/4/InvertTranslation /Settings/DigitalInput/4/Multiplier /Settings/DigitalInput/4/Type /Settings/DigitalInput/5/AlarmSetting /Settings/DigitalInput/5/Count /Settings/DigitalInput/5/CustomName /Settings/DigitalInput/5/InvertAlarm /Settings/DigitalInput/5/InvertTranslation /Settings/DigitalInput/5/Multiplier /Settings/DigitalInput/5/Type /Settings/DigitalInput/6/AlarmSetting /Settings/DigitalInput/6/Count /Settings/DigitalInput/6/CustomName /Settings/DigitalInput/6/InvertAlarm /Settings/DigitalInput/6/InvertTranslation /Settings/DigitalInput/6/Multiplier /Settings/DigitalInput/6/Type /Settings/FischerPanda0/AcLoad/Enabled /Settings/FischerPanda0/AcLoad/Measurement /Settings/FischerPanda0/AcLoad/QuietHoursStartValue /Settings/FischerPanda0/AcLoad/QuietHoursStopValue /Settings/FischerPanda0/AcLoad/StartTimer /Settings/FischerPanda0/AcLoad/StartValue /Settings/FischerPanda0/AcLoad/StopTimer /Settings/FischerPanda0/AcLoad/StopValue /Settings/FischerPanda0/Alarms/NoGeneratorAtAcIn /Settings/FischerPanda0/AutoStartEnabled /Settings/FischerPanda0/BatteryCurrent/Enabled /Settings/FischerPanda0/BatteryCurrent/QuietHoursStartValue /Settings/FischerPanda0/BatteryCurrent/QuietHoursStopValue /Settings/FischerPanda0/BatteryCurrent/StartTimer /Settings/FischerPanda0/BatteryCurrent/StartValue /Settings/FischerPanda0/BatteryCurrent/StopTimer /Settings/FischerPanda0/BatteryCurrent/StopValue /Settings/FischerPanda0/BatteryService /Settings/FischerPanda0/BatteryVoltage/Enabled /Settings/FischerPanda0/BatteryVoltage/QuietHoursStartValue /Settings/FischerPanda0/BatteryVoltage/QuietHoursStopValue /Settings/FischerPanda0/BatteryVoltage/StartTimer /Settings/FischerPanda0/BatteryVoltage/StartValue /Settings/FischerPanda0/BatteryVoltage/StopTimer /Settings/FischerPanda0/BatteryVoltage/StopValue /Settings/FischerPanda0/InverterHighTemp/Enabled /Settings/FischerPanda0/InverterHighTemp/StartTimer /Settings/FischerPanda0/InverterHighTemp/StopTimer /Settings/FischerPanda0/InverterOverload/Enabled /Settings/FischerPanda0/InverterOverload/StartTimer /Settings/FischerPanda0/InverterOverload/StopTimer /Settings/FischerPanda0/MinimumRuntime /Settings/FischerPanda0/OnLossCommunication /Settings/FischerPanda0/QuietHours/EndTime /Settings/FischerPanda0/QuietHours/StartTime /Settings/FischerPanda0/Soc/Enabled /Settings/FischerPanda0/Soc/QuietHoursStartValue /Settings/FischerPanda0/Soc/QuietHoursStopValue /Settings/FischerPanda0/Soc/StartValue /Settings/FischerPanda0/Soc/StopValue /Settings/FischerPanda0/StopWhenAc1Available /Settings/FischerPanda0/TestRun/Duration /Settings/FischerPanda0/TestRun/Enabled /Settings/FischerPanda0/TestRun/Interval /Settings/FischerPanda0/TestRun/RunTillBatteryFull /Settings/FischerPanda0/TestRun/SkipRuntime /Settings/FischerPanda0/TestRun/StartDate /Settings/FischerPanda0/TestRun/StartTime /Settings/Generator0/AcLoad/Enabled /Settings/Generator0/AcLoad/Measurement /Settings/Generator0/AcLoad/QuietHoursStartValue /Settings/Generator0/AcLoad/QuietHoursStopValue /Settings/Generator0/AcLoad/StartTimer /Settings/Generator0/AcLoad/StartValue /Settings/Generator0/AcLoad/StopTimer /Settings/Generator0/AcLoad/StopValue /Settings/Generator0/Alarms/NoGeneratorAtAcIn /Settings/Generator0/AutoStartEnabled /Settings/Generator0/BatteryCurrent/Enabled /Settings/Generator0/BatteryCurrent/QuietHoursStartValue /Settings/Generator0/BatteryCurrent/QuietHoursStopValue /Settings/Generator0/BatteryCurrent/StartTimer /Settings/Generator0/BatteryCurrent/StartValue /Settings/Generator0/BatteryCurrent/StopTimer /Settings/Generator0/BatteryCurrent/StopValue /Settings/Generator0/BatteryService /Settings/Generator0/BatteryVoltage/Enabled /Settings/Generator0/BatteryVoltage/QuietHoursStartValue /Settings/Generator0/BatteryVoltage/QuietHoursStopValue /Settings/Generator0/BatteryVoltage/StartTimer /Settings/Generator0/BatteryVoltage/StartValue /Settings/Generator0/BatteryVoltage/StopTimer /Settings/Generator0/BatteryVoltage/StopValue /Settings/Generator0/InverterHighTemp/Enabled /Settings/Generator0/InverterHighTemp/StartTimer /Settings/Generator0/InverterHighTemp/StopTimer /Settings/Generator0/InverterOverload/Enabled /Settings/Generator0/InverterOverload/StartTimer /Settings/Generator0/InverterOverload/StopTimer /Settings/Generator0/MinimumRuntime /Settings/Generator0/OnLossCommunication /Settings/Generator0/QuietHours/Enabled /Settings/Generator0/QuietHours/EndTime /Settings/Generator0/QuietHours/StartTime /Settings/Generator0/Soc/Enabled /Settings/Generator0/Soc/QuietHoursStartValue /Settings/Generator0/Soc/QuietHoursStopValue /Settings/Generator0/Soc/StartValue /Settings/Generator0/Soc/StopValue /Settings/Generator0/StopWhenAc1Available /Settings/Generator0/TestRun/Duration /Settings/Generator0/TestRun/Enabled /Settings/Generator0/TestRun/Interval /Settings/Generator0/TestRun/RunTillBatteryFull /Settings/Generator0/TestRun/SkipRuntime /Settings/Generator0/TestRun/StartDate /Settings/Generator0/TestRun/StartTime /Settings/Gps/Format /Settings/Gps/SpeedUnit /Settings/Gui/AutoBrightness /Settings/Gui/Brightness /Settings/Gui/DisplayOff /Settings/Gui/Language /Settings/Gui/MobileOverview /Settings/Gui/StartWithMenuView /Settings/Gui/TanksOverview /Settings/GuiMods/AcCurrentLimit/Preset1 /Settings/GuiMods/AcCurrentLimit/Preset2 /Settings/GuiMods/AcCurrentLimit/Preset3 /Settings/GuiMods/AcCurrentLimit/Preset4 /Settings/GuiMods/CustomDcSystemName /Settings/GuiMods/EnhancedFlowCombineLoads /Settings/GuiMods/FlowOverview /Settings/GuiMods/GaugeLimits/AcOutputMaxPower /Settings/GuiMods/GaugeLimits/AcOutputNonCriticalMaxPower /Settings/GuiMods/GaugeLimits/AlternatorMaxPower /Settings/GuiMods/GaugeLimits/BatteryMaxChargeCurrent /Settings/GuiMods/GaugeLimits/BatteryMaxDischargeCurrent /Settings/GuiMods/GaugeLimits/CautionPower /Settings/GuiMods/GaugeLimits/ContiuousPower /Settings/GuiMods/GaugeLimits/DcSystemMaxCharge /Settings/GuiMods/GaugeLimits/DcSystemMaxLoad /Settings/GuiMods/GaugeLimits/MaxAcChargerPower /Settings/GuiMods/GaugeLimits/MaxAlternatorPower /Settings/GuiMods/GaugeLimits/MaxChargerPower /Settings/GuiMods/GaugeLimits/MaxFeedInPower /Settings/GuiMods/GaugeLimits/MaxFuelCellPower /Settings/GuiMods/GaugeLimits/MaxWindGenPower /Settings/GuiMods/GaugeLimits/PeakPower /Settings/GuiMods/GaugeLimits/PvChargerMaxPower /Settings/GuiMods/GaugeLimits/PvOnGridMaxPower /Settings/GuiMods/GaugeLimits/PvOnOutputMaxPower /Settings/GuiMods/MoveSettings /Settings/GuiMods/ShortenTankNames /Settings/GuiMods/ShowBatteryTempOnFlows /Settings/GuiMods/ShowEnhancedFlowLoadsOnInput /Settings/GuiMods/ShowEnhancedFlowOverviewTanks /Settings/GuiMods/ShowEnhancedFlowOverviewTemps /Settings/GuiMods/ShowGauges /Settings/GuiMods/ShowInactiveFlowTiles /Settings/GuiMods/ShowRelayOverview /Settings/GuiMods/ShowTanksTempsDigIn /Settings/GuiMods/ShowTileOverview /Settings/GuiMods/TemperatureScale /Settings/GuiMods/TimeFormat /Settings/GuiMods/UseEnhancedFlowOverview /Settings/GuiMods/UseEnhancedGridParallelFlowOverview /Settings/GuiMods/UseEnhancedMobileOverview /Settings/GuiMods/UsedEnhancedGeneratorOverview /Settings/PackageManager/AutoInstall /Settings/PackageManager/Count /Settings/PackageManager/GitHubAutoDownload /Settings/Pump0/AutoStartEnabled /Settings/Pump0/Mode /Settings/Pump0/StartValue /Settings/Pump0/StopValue /Settings/Pump0/TankService /Settings/Relay/0/CustomName /Settings/Relay/0/InitialState /Settings/Relay/0/Show /Settings/Relay/1/CustomName /Settings/Relay/1/Function /Settings/Relay/1/InitialState /Settings/Relay/1/Polarity /Settings/Relay/1/Show /Settings/Relay/2/CustomName /Settings/Relay/2/InitialState /Settings/Relay/2/Show /Settings/Relay/3/CustomName /Settings/Relay/3/InitialState /Settings/Relay/3/Show /Settings/Relay/4/CustomName /Settings/Relay/4/InitialState /Settings/Relay/4/Show /Settings/Relay/5/CustomName /Settings/Relay/5/InitialState /Settings/Relay/5/Show /Settings/Relay/6/InitialState /Settings/Relay/Function /Settings/Relay/Polarity /Settings/Services/AccessPoint /Settings/Services/BleSensors /Settings/Services/Bluetooth /Settings/Services/Bol /Settings/Services/Console /Settings/Services/FischerPandaAutoStartStop /Settings/Services/Modbus /Settings/Services/MqttLocal /Settings/Services/MqttLocalInsecure /Settings/Services/MqttN2k /Settings/Services/MqttVrm /Settings/Services/NodeRed /Settings/Services/SignalK /Settings/ShutdownMonitor/ExternalSwitch /Settings/System/AccessLevel /Settings/System/AutoUpdate /Settings/System/ImageType /Settings/System/LogLevel /Settings/System/ReleaseType /Settings/System/RemoteSupport /Settings/System/SSHLocal /Settings/System/TimeZone /Settings/System/Units/Temperature /Settings/System/VncInternet /Settings/System/VncLocal /Settings/System/VolumeUnit /Settings/SystemSetup/AcInput1 /Settings/SystemSetup/AcInput2 /Settings/SystemSetup/BatteryService /Settings/SystemSetup/HasAcOutSystem /Settings/SystemSetup/HasDcSystem /Settings/SystemSetup/MaxChargeCurrent /Settings/SystemSetup/MaxChargeVoltage /Settings/SystemSetup/SharedTemperatureSense /Settings/SystemSetup/SharedVoltageSense /Settings/SystemSetup/SystemName /Settings/SystemSetup/TemperatureService /Settings/Vrmlogger/HttpsEnabled /Settings/Vrmlogger/LogInterval /Settings/Vrmlogger/Logmode /Settings/Vrmlogger/RamDiskMode /Settings/Vrmlogger/Url /Settings/Watchdog/VrmTimeout /Settings/Devices/adc_builtin0_1/Standard2 /Settings/Devices/adc_builtin0_1/Alarms/High/Active /Settings/Devices/adc_builtin0_1/Alarms/High/Delay /Settings/Devices/adc_builtin0_1/Alarms/High/Enable /Settings/Devices/adc_builtin0_1/Alarms/High/Restore /Settings/Devices/adc_builtin0_1/Alarms/Low/Active /Settings/Devices/adc_builtin0_1/Alarms/Low/Delay /Settings/Devices/adc_builtin0_1/Alarms/Low/Enable /Settings/Devices/adc_builtin0_1/Alarms/Low/Restore /Settings/Devices/adc_builtin0_1/Capacity /Settings/Devices/adc_builtin0_1/CustomName /Settings/Devices/adc_builtin0_1/FilterLength /Settings/Devices/adc_builtin0_1/FluidType2 /Settings/Devices/adc_builtin0_1/Function /Settings/Devices/adc_builtin0_1/RawValueEmpty /Settings/Devices/adc_builtin0_1/RawValueFull /Settings/Devices/adc_builtin0_1/Shape /Settings/Devices/adc_builtin0_2/Standard2 /Settings/Devices/adc_builtin0_2/Alarms/High/Active /Settings/Devices/adc_builtin0_2/Alarms/High/Delay /Settings/Devices/adc_builtin0_2/Alarms/High/Enable /Settings/Devices/adc_builtin0_2/Alarms/High/Restore /Settings/Devices/adc_builtin0_2/Alarms/Low/Active /Settings/Devices/adc_builtin0_2/Alarms/Low/Delay /Settings/Devices/adc_builtin0_2/Alarms/Low/Enable /Settings/Devices/adc_builtin0_2/Alarms/Low/Restore /Settings/Devices/adc_builtin0_2/Capacity /Settings/Devices/adc_builtin0_2/CustomName /Settings/Devices/adc_builtin0_2/FilterLength /Settings/Devices/adc_builtin0_2/FluidType2 /Settings/Devices/adc_builtin0_2/Function /Settings/Devices/adc_builtin0_2/RawValueEmpty /Settings/Devices/adc_builtin0_2/RawValueFull /Settings/Devices/adc_builtin0_2/Shape /Settings/Devices/adc_builtin0_3/Standard2 /Settings/Devices/adc_builtin0_3/Alarms/High/Active /Settings/Devices/adc_builtin0_3/Alarms/High/Delay /Settings/Devices/adc_builtin0_3/Alarms/High/Enable /Settings/Devices/adc_builtin0_3/Alarms/High/Restore /Settings/Devices/adc_builtin0_3/Alarms/Low/Active /Settings/Devices/adc_builtin0_3/Alarms/Low/Delay /Settings/Devices/adc_builtin0_3/Alarms/Low/Enable /Settings/Devices/adc_builtin0_3/Alarms/Low/Restore /Settings/Devices/adc_builtin0_3/Capacity /Settings/Devices/adc_builtin0_3/CustomName /Settings/Devices/adc_builtin0_3/FilterLength /Settings/Devices/adc_builtin0_3/FluidType2 /Settings/Devices/adc_builtin0_3/Function /Settings/Devices/adc_builtin0_3/RawValueEmpty /Settings/Devices/adc_builtin0_3/RawValueFull /Settings/Devices/adc_builtin0_3/Shape /Settings/Devices/adc_builtin0_4/Standard2 /Settings/Devices/adc_builtin0_4/Alarms/High/Active /Settings/Devices/adc_builtin0_4/Alarms/High/Delay /Settings/Devices/adc_builtin0_4/Alarms/High/Enable /Settings/Devices/adc_builtin0_4/Alarms/High/Restore /Settings/Devices/adc_builtin0_4/Alarms/Low/Active /Settings/Devices/adc_builtin0_4/Alarms/Low/Delay /Settings/Devices/adc_builtin0_4/Alarms/Low/Enable /Settings/Devices/adc_builtin0_4/Alarms/Low/Restore /Settings/Devices/adc_builtin0_4/Capacity /Settings/Devices/adc_builtin0_4/CustomName /Settings/Devices/adc_builtin0_4/FilterLength /Settings/Devices/adc_builtin0_4/FluidType2 /Settings/Devices/adc_builtin0_4/Function /Settings/Devices/adc_builtin0_4/Shape /Settings/Devices/adc_builtin0_4/RawValueEmpty /Settings/Devices/adc_builtin0_4/RawValueFull /Settings/Devices/adc_builtin0_5/CustomName /Settings/Devices/adc_builtin0_5/FilterLength /Settings/Devices/adc_builtin0_5/Function /Settings/Devices/adc_builtin0_5/Offset /Settings/Devices/adc_builtin0_5/Scale /Settings/Devices/adc_builtin0_5/TemperatureType2 /Settings/Devices/adc_builtin0_6/CustomName /Settings/Devices/adc_builtin0_6/FilterLength /Settings/Devices/adc_builtin0_6/Function /Settings/Devices/adc_builtin0_6/Offset /Settings/Devices/adc_builtin0_6/Scale /Settings/Devices/adc_builtin0_6/TemperatureType2 /Settings/Devices/adc_builtin0_7/CustomName /Settings/Devices/adc_builtin0_7/FilterLength /Settings/Devices/adc_builtin0_7/Function /Settings/Devices/adc_builtin0_7/Offset /Settings/Devices/adc_builtin0_7/Scale /Settings/Devices/adc_builtin0_7/TemperatureType2 /Settings/Devices/adc_builtin0_8/CustomName /Settings/Devices/adc_builtin0_8/FilterLength /Settings/Devices/adc_builtin0_8/Function /Settings/Devices/adc_builtin0_8/Offset /Settings/Devices/adc_builtin0_8/Scale /Settings/Devices/adc_builtin0_8/TemperatureType2 /Settings/TempSensorRelay/adc_builtin0_5/0/ClearValue /Settings/TempSensorRelay/adc_builtin0_5/0/Relay /Settings/TempSensorRelay/adc_builtin0_5/0/SetValue /Settings/TempSensorRelay/adc_builtin0_5/1/ClearValue /Settings/TempSensorRelay/adc_builtin0_5/1/Relay /Settings/TempSensorRelay/adc_builtin0_5/1/SetValue /Settings/TempSensorRelay/adc_builtin0_5/Enabled /Settings/TempSensorRelay/adc_builtin0_6/0/ClearValue /Settings/TempSensorRelay/adc_builtin0_6/0/Relay /Settings/TempSensorRelay/adc_builtin0_6/0/SetValue /Settings/TempSensorRelay/adc_builtin0_6/1/ClearValue /Settings/TempSensorRelay/adc_builtin0_6/1/Relay /Settings/TempSensorRelay/adc_builtin0_6/1/SetValue /Settings/TempSensorRelay/adc_builtin0_6/Enabled /Settings/TempSensorRelay/adc_builtin0_7/0/ClearValue /Settings/TempSensorRelay/adc_builtin0_7/0/Relay /Settings/TempSensorRelay/adc_builtin0_7/0/SetValue /Settings/TempSensorRelay/adc_builtin0_7/1/ClearValue /Settings/TempSensorRelay/adc_builtin0_7/1/Relay /Settings/TempSensorRelay/adc_builtin0_7/1/SetValue /Settings/TempSensorRelay/adc_builtin0_7/Enabled /Settings/TempSensorRelay/adc_builtin0_8/0/ClearValue /Settings/TempSensorRelay/adc_builtin0_8/0/Relay /Settings/TempSensorRelay/adc_builtin0_8/0/SetValue /Settings/TempSensorRelay/adc_builtin0_8/1/ClearValue /Settings/TempSensorRelay/adc_builtin0_8/1/Relay /Settings/TempSensorRelay/adc_builtin0_8/1/SetValue /Settings/TempSensorRelay/adc_builtin0_8/Enabled ================================================ FILE: setup ================================================ #!/bin/bash # SetupHelper provides a set of utilities used by other packages to streamline installing and removing packages # and facilitates reinstallation following a Venus OS update # package setup scripts can be run from the command line for manual installation and uninstallation, # and in some cases inputing configuration options # # SetupHelper includes PackageManager which manages package updates from GitHub # as well as package installation an uninstallation from the main GUI # # this setup script does NOT use version-dependent file sets # rather it makes modifications as part of this script # so that updates are not required when Venus OS versions are added # tell CommonResources to: # prompt for install/uninstall # auto install or auto uninstall # then exit # CommonResources will NOT return here ! standardPromptAndActions='yes' #rebootNeeded=true #### following line incorporates helper resources into this script source "/data/SetupHelper/HelperResources/IncludeHelpers" #### end of lines to include helper resources ================================================ FILE: updatePackage ================================================ #!/bin/bash # this script updates the package contents for all packages specified on the command line # or in the list below if 'all' is specified # file sets are updated/created for all Venus OS versions in the stockFiles directory # # NOTE: this script will create packages that will NOT work with SetupHelper prior to v6.0~2 # however helper resources are included in the package and are used instead of those provided by SH < v6.0~2 # the setup script for the package should source the local InstallHelpers file not CommonResources in SH # # if fileListVersionIndependent is present, # files are moved from the package directory to the VersionIndependent file set # # any .ALT_ORIG files are moved from the package directory to the AlternateOriginals directory # # replacement files may optionally be created from the original with a patch file # the files to be patched are listed in fileListPatched # patch files exist in FileSets/PatchSource # in order to create a patch file here, a source and edited file must also reside there # # patched replacement files are created in CommonResources prior to returning control to the setup script # # This is a unix bash script and should be run on a host computer, not a GX device # Windows will not run this script natively. # However Windows 10 apparently supports bash: # https://www.howtogeek.com/249966/how-to-install-and-use-the-linux-bash-shell-on-windows-10/ # # packages to be evaulated may be specified on the command line # use 'all' to process all packages in the allPackages list below # # file sets which contain real files (not just links and flags) # for a version NOT contained in StockFiles will be flagged with UNUSED_FILE_SET # this can occur if you remove versions from StockFiles. # For example, you may wish to remove beta versions after a beta test cycle. # file sets which do not contain any real files (just symbolic links or flag files) are removed # stockFiles contains excerpts from Venus OS file systems # and must be stored on the host # within a directory with name of the exact Venus OS version # and within the stockFiles directory defined below. # # 1) missing file set directories are created # 2) if any files in fileList don't exist (eg, a new file was added to file list), # the original file in stockFiles is copied to the version directory # 3) if the original file does not exist, the file is so marked with .NO_STOCK_FILE # this situation must be corrected: # version-dependent files without an original MUST use an "alternate original" # specified in FileSets. This permits version checks for these files too # replacement files that do not replace a stock file should be placed in version-indpendent file storage # (FileSets/). # If these replacement files vary with Venus OS versions, they MUST include an "alternate original". # This permits version checks for these files too # # when a stock file set does not exist, this script will check files from existing file sets # for a matching original file. # If a match is found, the replacement file is automatically placed in the new file set # If no match is found, the missing replacement is flagged and a suitable replacement must be created manually. # # existing file sets not in the stockFiles are checked. If empty, they are removed. # If not empty they are marked UNUSED_FILE_SET and flagged for manual removal. # # file sets will include all files listed in fileList. # this allows the setup script to always have a replacement for known versions # without searching other file sets for a matching original file # (there have been cases where installation fails because the search for a matching original could not be found) # if the stock file matches a previous version, a symbolic link for the replacement is created # rather than duplicating the file # this also makes maintanence easier since matching replacement can be identified # # original files in the file set are not normally used when installing the package # however, they are retained so that the setup script can attempt to create a file set for an unknown Venus OS version # this of course may fail if a matching original file can not be found # # if no end action is specified on the command line, the user is prompted for how to proceed for each package processed # end actions specified will bypass this prompt and proceed with the next package # end actions: # -p do not update the package but preserve the working copy # -d do not update the package and deete the working copy # -u update the package with changes in the working copy's file sets # # -r restore package from backup if present - no processing is performed on the packge, backup or working copy # backups are automatically created when updating a package # Note: the restore option is not offered at the end prompt since the update has not been applied yet. # # if errors occur, the needed corrections may be more obvious by comparing the package and the working copy # for this reason, preserving the working copy is recommended if errors are expected # set allPackages to all packages this script should evalueate if no options are included allPackages="SetupHelper TailscaleGX ShutdownMonitor VeCanSetup RpiDisplaySetup RpiGpioSetup GuiMods" ## ExtTransferSwitch GeneratorConnector TankRepeater are obsolete and file sets should not be updated. # attempt to locate SharedUtilities based on the location of this script # (it is assumed to be in the SetupHelper directory) # also sets the package root directory based on this also # and also the stock files base directory # # if these are not correct, edit the lines below to set the appropriate values scriptDir="$( cd $(dirname "$0") >/dev/null 2>&1 ; /bin/pwd -P )" packageRoot="$( dirname $scriptDir )" stockFiles="$packageRoot/StockVenusOsFiles" #### set these as appropriate to your system if the values set above are not correct #### packageRoot=FILL_THIS_IN_AND_UNCOMMENT_LINE #### stockFiles=FILL_THIS_IN_AND_UNCOMMENT_LINE if [ ! -e "$packageRoot" ]; then echo "unable to locate package root - can't continue" exit elif [ ! -e "$stockFiles" ]; then echo "unable to locate stock files - can't continue" exit fi # convert a version string to an integer to make comparisions easier # # Note: copied from VersionResources # but also includes code to report duplcates not in the VersionResources version function versionStringToNumber () { local version="$*" local numberParts local versionParts local numberParts local otherParts local other local number=0 local type='release' # split incoming string into # an array of numbers: major, minor, prerelease, etc # and an array of other substrings # the other array is searched for releasy type strings and the related offest added to the version number read -a numberParts <<< $(echo $version | tr -cs '0-9' ' ') numberPartsLength=${#numberParts[@]} if (( $numberPartsLength == 0 )); then versionNumber=0 versionStringToNumberStatus="$version: invalid, missing major version" return 1 fi if (( $numberPartsLength >= 2 )); then read -a otherParts <<< $(echo $version | tr -s '0-9' ' ') for other in ${otherParts[@]}; do case $other in 'b' | '~') type='beta' (( number += 60000 )) break ;; 'a') type='alpha' (( number += 30000 )) break ;; 'd') type='develop' break ;; esac done fi # if release all parts contribute to the main version number # and offset is greater than all prerelease versions if [ "$type" == "release" ] ; then (( number += 90000 )) # if pre-release, last part will be the pre release part # and others part will be part the main version number else (( numberPartsLength-- )) (( number += 10#${numberParts[$numberPartsLength]} )) fi # include core version number (( number += 10#${numberParts[0]} * 10000000000000 )) if (( numberPartsLength >= 2)); then (( number += 10#${numberParts[1]} * 1000000000 )) fi if (( numberPartsLength >= 3)); then (( number += 10#${numberParts[2]} * 100000 )) fi versionNumber=$number versionStringToNumberStatus="$version:$number $type" return 0 } # getFileLists reads the file list from files in the FileSets directory # # 'fileList' file must only list version-dependent files # 'fileListVersionIndependent' file must list only version-independent files # prior to SetupHelper v6.0, this list is ignored # # $1 specifies where the path to the fileList files # # three composite file lists are returned in global arrays: # fileList contains only version-dependent files # fileListVersionIndependent contains only version-independent files # fileListAll contains both versioned and version-independent files # # Note: copied from CommonResources function getFileLists () { local verListFile="$1/fileList" local indListFile="$1/fileListVersionIndependent" local patchListFile="$1/fileListPatched" local tempListVer=() local tempListInd=() local tempListPatched=() if [ -f "$verListFile" ]; then while read -r line || [[ -n "$line" ]]; do read -a params <<< $line # parse line into space-separted parameters then discard any that don't begin with / # this strips all comments beginning with # as well as any leading or trailing spaces for param in ${params[@]} ; do case $param in /*) tempListVer+=("$param") ;; esac done done < "$verListFile" fi if [ -f "$indListFile" ]; then while read -r line || [[ -n "$line" ]]; do read -a params <<< $line for param in ${params[@]} ; do case $param in /*) tempListInd+=("$param") ;; esac done done < "$indListFile" fi if [ -f "$patchListFile" ]; then while read -r line || [[ -n "$line" ]]; do read -a params <<< $line for param in ${params[@]} ; do case $param in /*) tempListPatched+=("$param") ;; esac done done < "$patchListFile" fi # remove duplicate files from each list fileList=($(printf "%s\n" "${tempListVer[@]}" | sort -u)) fileListVersionIndependent=($(printf "%s\n" "${tempListInd[@]}" | sort -u)) fileListPatched=($(printf "%s\n" "${tempListPatched[@]}" | sort -u)) tempListAll=(${fileList[@]}) tempListAll+=(${fileListVersionIndependent[@]}) tempListAll+=(${fileListPatched[@]}) fileListAll=($(printf "%s\n" "${tempListAll[@]}" | sort -u)) # report duplicates local dupsVer=($(printf "%s\n" "${tempListVer[@]}" | sort | uniq -d )) local dupsInd=($(printf "%s\n" "${tempListInd[@]}" | sort | uniq -d )) local dupsPatched=($(printf "%s\n" "${tempListPatched[@]}" | sort | uniq -d )) local dupsAll=($(printf "%s\n" "${tempListAll[@]}" | sort | uniq -d )) if [ ! -z "$dupsVer" ]; then for dup in "$dupsVer" ; do logMessage "WARNING $package: duplicate in fileList $dup - ignored" done fi if [ ! -z "$dupsInd" ]; then for dup in "$dupsInd" ; do logMessage "WARNING $package: duplicate in fileListVersionIndependent $dup - ignored" done fi if [ ! -z "$dupsPatched" ]; then for dup in "$dupsPatched" ; do logMessage "WARNING $package: duplicate in fileListPatched $dup - ignored" done fi if [ ! -z "$dupsAll" ]; then for dup in "$dupsAll" ; do logMessage "ERROR $package: duplicate in combined file lists $dup - can't continue" done exit fi } totalErrors=0 totalWarnings=0 packageErrors=0 packageWarnings=0 outputtingProgress=false function logMessage () { if $outputtingProgress ; then clearProgress fi echo "$*" if [[ "$*" == "ERROR"* ]]; then ((totalErrors++)) ((packageErrors++)) elif [[ "$*" == "WARNING"* ]]; then ((totalWarnings++)) ((packageWarnings++)) fi } function outputProgressTick () { if ! $outputtingProgress ; then echo -en "$beginProgressString" fi echo -en "$1" outputtingProgress=true } function clearProgress () { # start a new line if outputting ticks if $outputtingProgress; then echo # echo -ne "\r\033[2K" #### erase line fi outputtingProgress=false } beginProgressString="" function beginProgress () { # erase the line but stay on it if $outputtingProgress ; then clearProgress fi if [ ! -z "$1" ]; then beginProgressString="$1 " echo -en "$beginProgressString" outputtingProgress=true fi } # removing a nested set of directories sometimes results in permission denied the first time # so try several times to be sure function deleteNestedDirectories () { rm -rf "$1" &> /dev/null if [ -d "$1" ] ; then rm -rf "$1" &> /dev/null if [ -d "$1" ] ; then rm -rf "$1" fi fi } yesNoPrompt () { response='' while true; do /bin/echo -n "$*" read response case $response in [yY]*) return 0 break ;; [nN]*) return 1 break ;; *) esac done } #### script code begins here packageList="" doAllPackages=false globalEndAction='' for param in $* ; do case $param in -[pP]*) logMessage "working copies will be preserved - packages will not be updated" globalEndAction='preserve' ;; -[dD]*) logMessage "working copies will be deleted - packages will not be updated" globalEndAction='delete' ;; -[uU]*) logMessage "packages will be updated after updating" globalEndAction='update' ;; -[rR]*) logMessage "packages will be restored from backups" globalEndAction='restore' ;; all) doAllPackages=true ;; *) packageList+=" "$1 esac shift done if $doAllPackages ; then packageList=$allPackages elif [ -z "$packageList" ]; then logMessage "ERROR no packages specified - use 'all' for all packages" exit fi if [ "$globalEndAction" == "restore" ]; then for package in $packageList; do sourceDirectory="$packageRoot/$package" sourceFiles="$sourceDirectory/FileSets" backupDirectory="$packageRoot/$package.backup" backupFiles="$backupDirectory/FileSets" sourceVeLib="$sourceDirectory/velib_python" backupVeLib="$backupDirectory/velib_python" if [ ! -d "$backupDirectory" ]; then logMessage "WARNING $package: no backup found - package NOT restored" continue fi logMessage "WARNING $package: restored from backup" deleteNestedDirectories "$sourceFiles" deleteNestedDirectories "$sourceVeLib" mv "$backupFiles" "$sourceFiles" if [ -e "$backupVeLib" ]; then mv -f "$backupVeLib" "$sourceVeLib" fi #### TODO: delayed implementaiton ####if [ -e "$backupDirectory/validFirmwareVersions" ]; then #### mv -f "$backupDirectory/validFirmwareVersions" "$sourceDirectory/validFirmwareVersions" ####fi deleteNestedDirectories $backupDirectory done exit fi # get helper resources version for later if [ -f "$packageRoot/SetupHelper/version" ]; then shVersion=$( cat "$packageRoot/SetupHelper/version" ) versionStringToNumber $shVersion shVersionNumber=$versionNumber else shVersion="" shVersionNumber=0 fi # make the version list from the directories in stock files # version lists are sorted so the most recent version is first tempList=() stockVersionList=($(ls -d "$stockFiles"/v[0-9]* 2> /dev/null)) for entry in ${stockVersionList[@]} ; do version=$(basename $entry) versionFile="$stockFiles/$version/opt/victronenergy/version" if [ -f "$versionFile" ]; then realVersion=$(cat "$versionFile" | head -n 1) else logMessage "ERROR version file missing from stock files $version - can't continue" exit fi if [ $version != $realVersion ]; then logMessage "ERROR $version name does not mactch Venus $realVersion - can't continue" exit fi if versionStringToNumber $version ; then tempList+=("$version:$versionNumber") else logMessage "ERROR invalid version $versionStringToNumberStatus - not added to list" fi done stockVersionList=( $(echo ${tempList[@]} | tr ' ' '\n' | sort -t ':' -r -n -k 2 | uniq ) ) stockVersionListLength=${#stockVersionList[@]} for package in $packageList; do packageErrors=0 packageWarnings=0 sourceDirectory="$packageRoot/$package" sourceFiles="$sourceDirectory/FileSets" workingDirectory="$packageRoot/$package.copy" workingFiles="$workingDirectory/FileSets" backupDirectory="$packageRoot/$package.backup" backupFiles="$backupDirectory/FileSets" versionIndependentFileSet="$workingFiles/VersionIndependent" sourceVeLib="$sourceDirectory/velib_python" workingVeLib="$workingDirectory/velib_python" backupVeLib="$backupDirectory/velib_python" if [ ! -d "$sourceDirectory" ] || [ ! -f "$sourceDirectory/version" ]; then logMessage "WARNING: $sourceDirectory - not a package directory - skipping" continue # next package fi if [ ! -d "$sourceFiles" ]; then logMessage "$package: no file sets" fi if ! [ -f "$sourceFiles/fileList" ]; then logMessage "$package: no version-dependent files" fi if ! [ -f "$sourceFiles/fileListVersionIndependent" ]; then logMessage "$package: no version-independent files" fi if ! [ -f "$sourceFiles/fileListPatched" ]; then logMessage "$package: no patches" fi # validate package version number tempVersion=$(cat "$sourceDirectory/version") if ! versionStringToNumber $tempVersion ; then logMessage "ERROR $package: version $versionStringToNumberStatus - skipping package" continue # next package fi # compute compatible version range - use values in original package if [ -f "$sourceDirectory/obsoleteVersion" ]; then obsoleteVersio=$(cat "$sourceDirectory/obsoleteVersion") if versionStringToNumber $obsoleteVersio ; then obsoleteVersionNumber=$versionNumber else logMessage "ERROR $package obsoleteVersion $versionStringToNumberStatus - skipping package" continue # next package fi else obsoleteVersionNumber=9999999999999999 fi if [ -f "$sourceDirectory/firstCompatibleVersion" ]; then firstCompatibleVersion=$(cat "$sourceDirectory/firstCompatibleVersion") # limit packages to v3.10 and newer else firstCompatibleVersion='v3.10' fi if versionStringToNumber $firstCompatibleVersion ; then firstVersionNumber=$versionNumber else logMessage "ERROR $package: firstCompatibleVersion $versionStringToNumberStatus - skipping package" continue # next package fi # make copy of source package FileSets replaceCopy=false if [ -e "$workingDirectory" ]; then logMessage "$(basename $workingDirectory) already exists" if yesNoPrompt " replace it (y) or continue updating the copy (n)? " ; then replaceCopy=true else logMessage "$package: checking existing working copy" fi fi if $replaceCopy || ! [ -e "$workingDirectory" ]; then if $replaceCopy ;then logMessage "$package: replacing working copy" else logMessage "$package: making working copy" fi deleteNestedDirectories "$workingDirectory" mkdir -p "$workingDirectory" if [ -d "$sourceFiles" ]; then cp -pR "$sourceFiles" "$workingFiles" fi if [ -d "$sourceVeLib" ]; then cp -pR "$sourceVeLib" "$workingVeLib" fi fi # clean up flag files from a previous run rm -f "$workingFiles"/*/INCOMPATIBLE_VERSION rm -f "$workingFiles"/*/UNUSED_FILE_SET rm -f "$workingFiles"/*/INCOMPLETE rm -f "$workingFiles"/*/COMPLETE rm -f "$workingFiles"/*/LINKS_ONLY rm -f "$workingFiles"/*/*.NO_ORIG rm -f "$workingFiles"/*/*.CHECK_REPLACEMENT rm -f "$workingFiles"/*/*.CHECK_PATCH rm -f "$workingFiles"/*/*.BAD_LINK rm -f "$workingFiles"/*/NEW_FILE_SET rm -f "$workingFiles"/*VERSIONED_AND_INDEPENDENT_EXIST rm -f "$workingFiles"/*/*VERSIONED_AND_INDEPENDENT_EXIST rm -f "$workingFiles"/*CHECK_VERSION_INDEPENDENT rm -f "$workingFiles"/*/*CHECK_VERSION_INDEPENDENT rm -f "$workingFiles"/*CHECK_ALT_ORIG rm -f "$workingFiles"/*/*CHECK_ALT_ORIG #### TODO: delayed implementaiton # create valid firmware version list ####rm -f "$workingDirectory/validFirmwareVersions" ####for (( i1 = 0; i1 < $stockVersionListLength; i1++ )); do #### IFS=':' read version versionNumber <<< "${stockVersionList[$i1]}" #### if (( versionNumber >= firstCompatibleVersionNumber )) && (( versionNumber < obsoleteVersionNumber )) ; then #### echo $version >> "$workingDirectory/validFirmwareVersions" #### fi ####done # update velib_python if [ -e "$workingVeLib" ]; then beginProgress "updating velib_python" pythonLibSoureDir="opt/victronenergy/dbus-systemcalc-py/ext/velib_python" veLibFiles=( vedbus.py dbusmonitor.py settingsdevice.py ve_utils.py ) rm -rf "$workingVeLib" mkdir "$workingVeLib" for (( i1 = 0; i1 < $stockVersionListLength; i1++ )); do newVersion=false IFS=':' read version versionNumber <<< "${stockVersionList[$i1]}" if (( i1 == 0 )); then newVersion=true else for file in ${veLibFiles[@]} ; do file1="$stockFiles/$version/$pythonLibSoureDir/$file" file2="$stockFiles/$previousVersion/$pythonLibSoureDir/$file" if ! cmp -s "$file1" "$file2" > /dev/null ; then newVersion=true fi done fi if $newVersion ; then outputProgressTick "." if (( i1 == 0 ));then velibDir="$workingVeLib/latest" prevVelibDir="$workingVeLib/latest" else velibDir="$workingVeLib/$version" fi mkdir "$velibDir" for file in ${veLibFiles[@]} ; do file1="$stockFiles/$version/$pythonLibSoureDir/$file" file2="$velibDir/$file" cp -f "$file1" "$file2" done newVersion=false previousVersion=$version prevVelibDir="$velibDir" fi echo $version > "$prevVelibDir/oldestVersion" done fi getFileLists "$workingFiles" # if any version-dependent files, create missing file sets or flag incompatible if ! [ -z $fileList ]; then for entry in ${stockVersionList[@]}; do IFS=':' read version versionNumber <<< "$entry" fileSet="$workingFiles/$version" stockFileSet="$stockFiles/$version" if (( $versionNumber >= $obsoleteVersionNumber )) || (( $versionNumber < $firstVersionNumber )); then touch "$fileSet/INCOMPATIBLE_VERSION" compatible=false else compatible=true fi if $compatible && ! [ -e "$fileSet" ]; then mkdir "$fileSet" touch "$fileSet/NEW_FILE_SET" fi done fi # add the package's existing file sets NOT in the stock versions and mark them unused # replacement files will not be moved to these unused file sets sourceFileSets=($(ls -d "$workingFiles"/v[0-9]* 2> /dev/null)) tempList=(${stockVersionList[@]}) for entry in ${sourceFileSets[@]} ; do version=$(basename $entry) if [ ! -d "$stockFiles/$version" ]; then if ! versionStringToNumber $version ; then logMessage "ERROR $package: file set name $versionStringToNumberStatus - can't continue" exit fi tempList+=($version:$versionNumber) touch "$workingFiles/$version/UNUSED_FILE_SET" fi done allFileSets=( $(echo ${tempList[@]} | tr ' ' '\n' | sort -t ':' -r -n -k 2 | uniq ) ) # move incompatible versions to the end of the list # so that real files end up in a supported file set obsoleteFileSets=() tempList=() for entry in ${allFileSets[@]} ; do IFS=':' read version versionNumber <<< "$entry" if (( $versionNumber >= $obsoleteVersionNumber )) || (( $versionNumber < $firstVersionNumber )); then obsoleteFileSets+=($version:$versionNumber) else tempList+=($version:$versionNumber) fi done allFileSets=(${tempList[@]}) allFileSets+=(${obsoleteFileSets[@]}) allFileSetsLength=${#allFileSets[@]} # relocate version-independent files to VersionIndependent file set for file in ${fileListVersionIndependent[@]}; do baseName=$(basename $file) if [ -f "$workingFiles/$baseName" ]; then if [ ! -d "$workingFiles/VersionIndependent" ]; then mkdir -p "$workingFiles/VersionIndependent" fi if [ -f "$workingFiles/VersionIndependent/$baseName" ]; then logMessage "$package: $baseName exists in FileSets AND VersionIndependent - not moved" touch "$workingFiles/VersionIndependent/$baseName.CHECK_VERSION_INDEPENDENT" touch "$workingFiles/$baseName.CHECK_VERSION_INDEPENDENT" else logMessage "$package: moving $baseName to version-independent file set" mv "$workingFiles/$baseName" "$workingFiles/VersionIndependent" fi elif ! [ -f "$workingFiles/VersionIndependent/$baseName" ]; then logMessage "ERROR $package: $baseName missing version independent file" fi done # relocate ALT_ORIG files to .../FileSets/AlternateOriginals # do as a loop so each move is reported # must have some alt orig files in FileSets for this to happen altOrigFileDir="$workingFiles/AlternateOriginals" oldAltOrigList=( $( ls "$workgingFiles"/*.ALT_ORIG 2> /dev/null) ) if ! [ -z "$oldAltOrigList" ] && yesNoPrompt "move alternate originals to AlternateOriginal directory (y/n)? " ; then if ! [ -d "$altOrigFileDir" ]; then mkdir -p "$altOrigFileDir" fi for file in ${altOrigList[@]} ; do baseName=$(basename "$file") if [ -f "$altOrigFileDir/$baseName" ]; then logMessage "$package: $baseName exists in FileSets AND AlternateOriginals - not moved" touch "$altOrigFileDir/$baseName.CHECK_ALT_ORIG" touch "$workingFiles/$baseName.CHECK_ALT_ORIG" else logMessage "$package: moving $baseName.ALT_ORIG to AlternateOriginals" mv "$file" "$altOrigFileDir" fi done fi # create and test patch files # the option to skip creation exists in case the patch file needs to be hand edited # # in order to create a patch file, both an unmodified file without the changes # and an 'result' file WITH the desired changes are required # # multiple sets can exist and are of the form # result $baseName* eg PageSettings.qml-1 # orig $baseName*.orig eg PageSettings.qml-1.orig # patch $baseName*.patch eg PageSettings.qml-1.patch # # legacy .edited and .source, will be converted to the new format patchSourceDir="$workingFiles/PatchSource" if (( ${#fileListPatched[@]} > 0 )); then beginProgress "checking patch files" for file in ${fileListPatched[@]} ; do outputProgressTick "." baseName=$( basename $file ) patchOptionsFile="$patchSourceDir/$baseName.patchOptions" if [ -f "$patchOptionsFile" ]; then patchOptions=$( cat "$patchOptionsFile" ) optionsModTime=$( date -r "$patchOptionsFile" '+%s' ) else patchOptions='-u' optionsModTime=0 fi # convert old name formats oldPath="$patchSourceDir/$baseName" newPath="$oldPath-1" if [ -e "$oldPath.source" ] || [ -e "$oldPath.edited" ]; then if [ -e "$newPath*" ]; then logMessage "WARNING $package: can't move .source and .edited patch files - ...-1... already exists" else if [ -e "$oldPath.source" ]; then logMessage "$package: renaming $baseName.source to $baseName.orig" mv "$oldPath.source" "$newPath.orig" fi if [ -e "$oldPath.edited" ]; then logMessage "$package: renaming $baseName.edited to $baseName" mv "$oldPath.edited" "$newPath" fi fi fi if [ "$patchOptions" == "MANUAL" ]; then logMessage "WARNING $package: $baseName patch options set to MANUAL - skipping patch update" else patchOrigFiles=( $( ls "$patchSourceDir/$baseName"*.orig 2> /dev/null ) ) for patchOrig in ${patchOrigFiles[@]}; do patchResult="${patchOrig/.orig/}" patchFile="${patchOrig/.orig/.patch}" if ! [ -f "$patchResult" ]; then logMessage "WARNING $package: missing $( basename $patchResult) - skipping patch update" continue fi diff $patchOptions "$patchOrig" "$patchResult" > "$patchFile" done fi # test each patch on all versions - need one to succeed for each version # both foward and reverse patches must succeed # and reverse patch result must match stock file patchFiles=( $( ls "$patchSourceDir/$baseName"*.patch 2> /dev/null ) ) for (( i1 = 0; i1 < $allFileSetsLength; i1++ )); do IFS=':' read version versionNumber <<< "${allFileSets[$i1]}" stockFile="$stockFiles/$version$file" replacement="$workingFiles/$version/$baseName" if ! [ -f "$stockFile" ]; then continue # check for case mismatch (main vs Main is NOT a match) else result=$( find $( dirname "$stockFile" ) -name $( basename "$stockFile" ) ) if [ "$result" == "" ]; then continue fi fi foundPatch=false for patchFile in ${patchFiles[@]}; do patchResultFile="$patchSourceDir/$baseName.patchResult" reversePatchResultFile="$patchSourceDir/$baseName.reversePatchResult" patchOk=true if ! yes 'no' | patch -N -o "$patchResultFile" "$stockFile" "$patchFile" > /dev/null ; then patchOk=false fi if $patchOK && ! yes 'no' | patch -R -o "$reversePatchResultFile" "$patchResultFile" "$patchFile" > /dev/null ; then patchOk=false fi if $patchOK && ! cmp -s "$stockFile" "$reversePatchResultFile" > /dev/null ; then patchOk=false fi rm -f "$patchResultFile"* "$reversePatchResultFile"* # found a patch that works - stop looking if $patchOk ; then foundPatch=true break fi done # if patch succeeded, remove any replacement and orig files in this file set # they won't be looked at below since fileList won't include them # checks above insure there the same file is not in multiple file lists if $foundPatch; then if [ -e "$replacement" ] && ! [ -L "$replacement" ] || [ -e "$replacement.USE_ORIGINAL" ]; then logMessage "WARNING $package: removing $baseName from $version file set" rm -f "$replacement"* fi else logMessage "ERROR $package: $version no patch file for $baseName" origFile="$patchSourceDir/$baseName-$version".orig if ! [ -e "$origFile" ]; then logMessage " adding $baseName-$version.orig - replacement must be created manually" cp "$stockFile" "$origFile" fi touch "$patchSourceDir/$baseName-$version".CHECK_REPLACEMENT touch "$patchSourceDir/$baseName-$version".CHECK_PATCH touch "$patchSourceDir/INCOMPLETE" fi done done # for file in fileListPatched fi # check patch files beginProgress "$package: updating file sets" # process only versioned files for file in ${fileList[@]} ; do outputProgressTick "." baseName=$(basename "$file") versionedFileExists=false # use alternate original if present in AlternateOriginals if [ -f "$altOrigFileDir/$baseName.ALT_ORIG" ]; then useAltOrig=true altOrigFile=$(cat "$altOrigFileDir/$baseName.ALT_ORIG") # or in FileSets elif [ -f "$workingFiles/$baseName.ALT_ORIG" ]; then useAltOrig=true altOrigFile=$(cat "$workingFiles/$baseName.ALT_ORIG") else useAltOrig=false altOrigFile="" fi # locate groups of file sets with matching stock (or .orig) files (( start = 0 )) while (( start < $allFileSetsLength )); do # locate one group of file sets that will use the same replacement file (( end = start )); (( to = -1 )) toFileSet="" compareReference="" startVersion="-" toVersion="-" endVersion="-" oldReplacementVersion="-" oldReplacementIsFile=false oldReplacementIsUseOrig=false addOrigToFileSet=false blockIsUsed=false for (( i1 = start; i1 < $allFileSetsLength; i1++ )); do IFS=':' read version versionNumber <<< "${allFileSets[$i1]}" fileSet="$workingFiles/$version" if [ -e "$fileSet/INCOMPATIBLE_VERSION" ] || [ -e "$fileSet/UNUSED_FILE_SET" ]; then fileSetUsed=false else fileSetUsed=true fi replacement="$fileSet/$baseName" stockFileSet="$stockFiles/$version" orig="$fileSet/$baseName.orig" if $useAltOrig ; then stockFile="$stockFileSet$altOrigFile" else stockFile="$stockFileSet$file" fi replacementIsLink=false replacementIsFile=false useOrig=false if [ -L "$replacement" ]; then replacementIsLink=true elif [ -f "$replacement" ]; then replacementIsFile=true elif [ -f "$replacement.USE_ORIGINAL" ]; then useOrig=true fi # skip this version if parent directory does not exist if ! [ -e $( dirname "$stockFile" ) ]; then fileSetUsed=false fi if (( i1 == start )); then startVersion=$version fi if ! $fileSetUsed ; then thisOrig="" elif [ -e "$stockFile" ]; then thisOrig="$stockFile" # error if file set exists but no stock file elif [ -e "$stockFileSet" ]; then if $useAltOrig ; then logMessage "ERROR $package: $version $baseName stock file missing - check ALT_ORIG - can't continue" touch "$fileSet/$baseName.CHECK_ALT_ORIG" else logMessage "ERROR $package: $version $baseName stock file missing - consider using an ALT_ORIG - can't continue" fi touch "$fileSet/$baseName.NO_STOCK_FILE" touch "$fileSet/INCOMPLETE" exit # no stock file - use orig in file set elif [ -e "$orig" ]; then thisOrig="$orig" else thisOrig="" fi if [ -z "$compareReference" ] && ! [ -z "$thisOrig" ]; then compareReference="$thisOrig" fi includeInBlock=false # nothing to compare - include in same block if $fileSetUsed ; then if [ -z "$compareReference" ] || [ -z "$thisOrig" ]; then includeInBlock=true elif [ "$thisOrig" == "$compareReference" ]; then includeInBlock=true # orig exists and DOES match others in block elif cmp -s "$thisOrig" "$compareReference" > /dev/null ; then includeInBlock=true fi fi # start a new block if ! $includeInBlock ; then break fi if $fileSetUsed ; then blockIsUsed=true fi # save version of old replacement files for next loop if $replacementIsFile ; then if ! $oldReplacementIsFile || [ "$oldReplacementVersion" == "-" ]; then oldReplacementVersion=$version oldReplacementIsFile=true oldReplacementIsUseOrig=false addOrigToFileSet=true fi elif $useOrig && [ "$oldReplacementVersion" == "-" ]; then oldReplacementVersion=$version oldReplacementIsFile=false oldReplacementIsUseOrig=true # retain .orig so that SetupHelper can create USE_ORIGINAL flag files for missing file sets addOrigToFileSet=true fi (( end = i1 )) endVersion=$version # first compatible file set - move replacements here # unused file sets are permitted for the destination file set # but if a used file set is later found, it is preferred if (( to == -1 )) && $fileSetUsed; then (( to = i1 )) toFileSet="$fileSet" toVersion=$( basename "$toFileSet" ) fi done # end locate block if (( to == -1 )); then if $blockIsUsed; then logMessage "ERROR $package: $baseName no destination file set for block $startVersion $endVersion - can't relocate files" fi else toReplacement="$toFileSet/$baseName" oldReplacement="$workingFiles/$oldReplacementVersion/$baseName" toStockFileSet="$stockFiles/$toVersion" if $useAltOrig ; then toStockFile="$toStockFileSet$altOrigFile" else toStockFile="$toStockFileSet$file" fi # relocate replacement & orig if $oldReplacementIsUseOrig ; then rm -f "$toReplacement"* touch "$toReplacement.USE_ORIGINAL" versionedFileExists=true elif $oldReplacementIsFile; then if [ "$oldReplacementVersion" != "$toVersion" ]; then rm -f "$toReplacement"* mv "$oldReplacement" "$toReplacement" fi versionedFileExists=true elif [ -e $( dirname "$toReplacement" ) ]; then touch "$toReplacement.CHECK_REPLACEMENT" fi if $addOrigToFileSet ; then if [ -f "$toStockFile" ]; then cp "$toStockFile" "$toReplacement.orig" elif [ -f "$oldReplacement.orig" ]; then mv "$oldReplacement.orig" "$toReplacement.orig" else logMessage "ERROR $package: $baseName no original for $toVersion" touch "$toReplacement.NO_ORIG" fi fi # update links and look for additional replacements for (( i1 = start; i1 <= end; i1++ )); do # skip to -- it's alredy updated if (( i1 == to )); then continue; fi IFS=':' read version versionNumber <<< "${allFileSets[$i1]}" fileSet="$workingFiles/$version" replacement="$fileSet/$baseName" replacementIsLink=false replacementIsFile=false useOrig=false if [ -L "$replacement" ]; then replacementIsLink=true elif [ -f "$replacement" ]; then replacementIsFile=true elif [ -f "$replacement.USE_ORIGINAL" ]; then useOrig=true fi if $useOrig ; then if ! $oldReplacementIsUseOrig ; then logMessage "WARNING $package: $baseName has replacement - USE_ORIGINAL removed" rm -f "$replacement.USE_ORIGINAL" fi fi # a second replacement is found - remove if matches, error if different updateReplacements=true if $replacementIsFile && $oldReplacementIsFile; then if ! cmp -s "$replacement" "$toReplacement" > /dev/null ; then logMessage "ERROR $package: $baseName $version second replacement differs from $toVersion - check replacements" touch "$replacement.CHECK_REPLACEMENT" touch "$toReplacement.CHECK_REPLACEMENT" updateReplacements=false fi fi if $updateReplacements ; then rm -f "$replacement"* if $oldReplacementIsUseOrig; then touch "$replacement.USE_ORIGINAL" # links are created even if replacement does not exist else ln -sf "../$toVersion/$baseName" "$replacement" fi fi done fi # continue looking for blocks with file set after the end of this block (( start = end + 1 )) done # while start ... # both versioned and version-independent files exist - report this (no action taken) if $versionedFileExists; then if [ -e "$versionIndependentFileSet/$baseName" ];then logMessage "WARNING $package: $baseName versioned file exists - version-independent file will be ignored" touch "$versionIndependentFileSet/$baseName.VERSIONED_AND_INDEPENDENT_EXIST" fi fi done # for file beginProgress "$package: final checks" for (( i1 = 0; i1 < $allFileSetsLength; i1++ )); do IFS=':' read version versionNumber <<< "${allFileSets[$i1]}" if (( $versionNumber >= $obsoleteVersionNumber )) || (( $versionNumber < $firstVersionNumber )); then incompatibleVersion=true else incompatibleVersion=false fi fileSet="$workingFiles/$version" # file set should exist so this shouldn't happen but report and skip checks anyway if ! [ -d "$fileSet" ]; then if ! $incompatibleVersion && ! [ -z $fileList ]; then logMessage "ERROR $package: $version missing file set - skipping" fi continue fi outputProgressTick "." replacementFilesExist=false for file in ${fileList[@]} ; do baseName=$(basename "$file") replacement="$fileSet/$baseName" if [ -f "$replacement" ] && ! [ -L "$replacement" ]; then replacementFilesExist=true break fi done # LINKS_ONLY is not used for anything but helps identify file sets that don't contain real files if ! $replacementFilesExist ; then touch "$fileSet/LINKS_ONLY" fi fileSetInUse=true # remove file sets for incompatible Venus OS versions if [ -e "$fileSet/INCOMPATIBLE_VERSION" ]; then fileSetInUse=false if $replacementFilesExist ; then logMessage "WARNING $package: $version not compatible with Venus $version but file set not empty - not removed" else if [ ! -f "$fileSet/NEW_FILE_SET" ]; then logMessage "WARNING $package: not compatible with Venus $version - file set removed" fi rm -Rf "$fileSet" fi # remove empty unused file sets elif [ -f "$fileSet/UNUSED_FILE_SET" ]; then fileSetInUse=false if $replacementFilesExist ; then logMessage "WARNING $package: $version no longer used but file set not empty - not removed" else # log removal of a previous file set if not created with this run # if it was created with this run, delete it silently if [ ! -f "$fileSet/NEW_FILE_SET" ]; then logMessage "WARNING $package: $version - removing unused file set" fi rm -Rf "$fileSet" fi fi # do final checks on versioned files only # and for file sets that still exist if [ -e "$fileSet" ] && $fileSetInUse; then if [ -e "$fileSet/NEW_FILE_SET" ]; then logMessage "$package: $version new file set" fi for file in ${fileList[@]} ; do baseName=$(basename "$file") # use alternate original if present in AlternateOriginals if [ -f "$altOrigFileDir/$baseName.ALT_ORIG" ]; then useAltOrig=true altOrigFile=$(cat "$altOrigFileDir/$baseName.ALT_ORIG") # or if in FileSets elif [ -f "$workingFiles/$baseName.ALT_ORIG" ]; then useAltOrig=true altOrigFile=$(cat "$workingFiles/$baseName.ALT_ORIG") else useAltOrig=false altOrigFile="" fi replacement="$fileSet/$baseName" orig="$fileSet/$baseName.orig" # missing replacement, check all other file sets once more if [ -f "$replacement.CHECK_REPLACEMENT" ]; then stockFileSet="$stockFiles/$version" if $useAltOrig ; then stockFile1="$stockFileSet$altOrigFile" else stockFile1="$stockFileSet$file" fi for (( i2 = 0; i2 < $allFileSetsLength; i2++ )); do if (( i2 == i1 )); then continue; fi IFS=':' read version2 versionNumber2 <<< "${allFileSets[$i2]}" fileSet2="$workingFiles/$version2" if ! [ -e "$fileSet2" ] || [ -e "$fileSet2.INCOMPATIBLE_VERSION" ]; then continue; fi replacement2="$fileSet2/$baseName" if ! [ -f "$replacement2" ] || [ -L "$replacement2" ]; then continue; fi orig2="$fileSet2/$baseName.orig" if ! [ -e "$orig2" ]; then continue; fi if cmp -s "$stockFile1" "$orig2" > /dev/null ; then ln -sf "../$version2/$baseName" "$replacement" rm -f "$replacement.CHECK_REPLACEMENT" rm -f "$orig" break fi done fi if [ -f "$replacement.CHECK_REPLACEMENT" ]; then enclosingDir=$( dirname "$stockFile1" ) if [ -e "$enclosingDir" ]; then dirName=$( basename "$enclosingDir" ) logMessage "ERROR $package "$dirName"/$baseName: no replacement for $version" # grab a copy of the orig file to make creating the replacement easier if ! [ -e "$replacement.orig" ]; then if [ -e "$stockFile1" ]; then cp "$stockFile1" "$replacement.orig" else logMessage "ERROR $package $"dirName"/$baseName: no stock file for $version" fi fi fi fi # validate sym link symLinkReplacement=false badLink=false if [ -L "$replacement" ]; then symLinkReplacement=true # resolive symlink then check to make sure that file is valid linkedFile=$( realpath "$replacement" 2> /dev/null ) if [ -z "$linkedFile" ]; then logMessage "ERROR $package: $baseName $version no linked file $linkedFile" touch "$replacement.BAD_LINK" else linkedFileSet=$( dirname "$linkedFile" ) if [ -z "$linkedFileSet" ]; then logMessage "ERROR $package: $baseName $version no linked file set" touch "$replacement.BAD_LINK" elif [ -f "$linkedFileSet/UNUSED_FILE_SET" ] || [ -f "$linkedFileSet/INCOMPATIBLE_VERSION" ]; then logMessage "ERROR $package: $baseName $version links to unused file set $linkedFileSet" touch "$replacement.BAD_LINK" fi fi fi # flag file set incomplete if [ -f "$replacement.CHECK_REPLACEMENT" ] || [ -f "$replacement.NO_ORIG" ] \ || [ -f "$replacement.BAD_LINK" ] ; then touch "$fileSet/INCOMPLETE" fi done # for file if [ -f "$fileSet/INCOMPLETE" ]; then rm -f "$fileSet/COMPLETE" else touch "$fileSet/COMPLETE" fi rm -f "$fileSet/NEW_FILE_SET" fi # do final checks for versioned ... done # for i1 (final checks) if (( $packageErrors == 0 )); then errorText="no errors " else errorText="$packageErrors ERRORS " fi if (( $packageWarnings == 0 )); then warningText="no warnings" else warningText="$packageWarnings WARNINGS" fi logMessage "$package complete $errorText $warningText" # report errors for file sets and patch files for (( i1 = 0; i1 < $allFileSetsLength; i1++ )); do IFS=':' read version versionNumber <<< "${allFileSets[$i1]}" fileSet="$workingFiles/$version" if ! [ -e "$fileSet" ]; then continue; fi # if all replacement files are in place, mark the file set COMPLETE # so _checkFileSets can skip all checks # COMPLETE tells _checkFileSets to skip all checks and accept the file set as is if [ -f "$fileSet/INCOMPLETE" ]; then rm -f "$fileSet/COMPLETE" logMessage " INCOMPLETE file set $version" else touch "$fileSet/COMPLETE" fi if [ -f "$fileSet/INCOMPATIBLE_VERSION" ]; then logMessage " INCOMPATIBLE VERSION $version" fi if [ -f "$fileSet/UNUSED_FILE_SET" ]; then logMessage " UNUSED file set $version" fi done if [ -f "$patchSourceDir/INCOMPLETE" ]; then logMessage " MISSING patch files" fi workingName=$(basename $workingDirectory) backupName=$(basename $backupDirectory) endAction="$globalEndAction" if [ "$endAction" == 'update' ] && (( $packageErrors != 0 )); then echo if ! yesNoPrompt "$package has errors - update anyway (y/n)? " ; then logMessage "changes preserved as $workingName" endAction='preserve' fi fi if [ -z "$endAction" ]; then echo echo "select to finish:" echo " update $package (u)" echo " preserve working copy (p)" echo " discard working copy (d)" while true ; do read -p "choose action from list above (u / p / d): " response case $response in [uU]*) if (( $packageErrors == 0 )); then endAction='update' else echo if yesNoPrompt "$package has errors - update anyway (y/n)? " ;then logMessage "updating $package (with errors)" endAction='update' else logMessage "changes preserved as $workingName" endAction='preserve' fi fi break ;; [pP]*) endAction='preserve' break ;; [dD]*) endAction='delete' break ;; *) esac done fi case $endAction in preserve) logMessage "$package not updated - changes preserved as $workingName" ;; delete) logMessage "$package not updated - $workingName removed" deleteNestedDirectories "$workingDirectory" ;; update) doBackup=true if [ -d "$backupDirectory" ] && ! yesNoPrompt "$backupName exists OVERWRITE it (y/n)? " ; then doBackup=false fi if $doBackup ; then logMessage "$package: $backupName updated" deleteNestedDirectories "$package.backup" mkdir "$backupDirectory" if [ -d "$sourceFiles" ]; then mv "$sourceFiles" "$backupFiles" fi sourceVeLib="$sourceDirectory/velib_python" backupVeLib="$backupDirectory/velib_python" if [ -d "$sourceVeLib" ]; then mv -f "$sourceVeLib" "$backupVeLib" fi #### TODO: delayed implementaiton ####if [ -e "$sourceDirectory/validFirmwareVersions" ]; then #### mv -f "$sourceDirectory/validFirmwareVersions" "$backupDirectory/validFirmwareVersions" ####fi else logMessage "$package: $backupName unchanged" fi logMessage "$package: updating package files" if [ -d "$workingFiles" ]; then deleteNestedDirectories "$sourceFiles" mv "$workingFiles" "$sourceFiles" fi if [ -e "$sourceDirectory/HelperResources" ] && [ "$package" != "SetupHelper" ]; then logMessage "$package: removing HelperResources" deleteNestedDirectories "$sourceDirectory/HelperResources" fi sourceVeLib="$sourceDirectory/velib_python" workingVeLib="$backupDirectory/velib_python" if [ -d "$workingVeLib" ]; then mv -f "$workingVeLib" "$sourceVeLib" fi #### TODO: delayed implementaiton ####if [ -e "$workingDirectory/validFirmwareVersions" ]; then #### mv -f "$workingDirectory/validFirmwareVersions" "$sourceDirectory/validFirmwareVersions" ####fi deleteNestedDirectories "$workingDirectory" ;; *) logMessage "ERROR: invalid end action $endAction" esac logMessage "" done # for package # review all file sets and report any that only contain sym links across all packages # it would be possile to remove those verions from stock files without loosing any data # this check is only done if updating all file sets and there are no errors if $doAllPackages && [ "$totalErrors" == 0 ]; then for entry in ${stockVersionList[@]} ; do IFS=':' read version versionNumber <<< "$entry" linksOnly=true for package in $packageList; do fileSet="$packageRoot/$package/FileSets/$version" if [ ! -e "$fileSet/LINKS_ONLY" ]; then linksOnly=false break fi done if $linksOnly ; then logMessage "$version: only links in all packages - stock version could be removed" fi done fi if [ "$totalErrors" == 0 ]; then errorText="no errors " else errorText="$totalErrors ERRORS " fi if [ "$totalWarnings" == 0 ]; then warningText="no warnings" else warningText="$totalWarnings WARNINGS" fi logMessage "updateFileSets complete $errorText $warningText" ================================================ FILE: velib_python/dbusmonitor.py ================================================ #!/usr/bin/env python3 # -*- coding: utf-8 -*- ## @package dbus_vrm # This code takes care of the D-Bus interface (not all of below is implemented yet): # - on startup it scans the dbus for services we know. For each known service found, it searches for # objects/paths we know. Everything we find is stored in items{}, and an event is registered: if a # value changes weĺl be notified and can pass that on to our owner. For example the vrmLogger. # we know. # - after startup, it continues to monitor the dbus: # 1) when services are added we do the same check on that # 2) when services are removed, we remove any items that we had that referred to that service # 3) if an existing services adds paths we update ourselves as well: on init, we make a # VeDbusItemImport for a non-, or not yet existing objectpaths as well1 # # Code is used by the vrmLogger, and also the pubsub code. Both are other modules in the dbus_vrm repo. from dbus.mainloop.glib import DBusGMainLoop from gi.repository import GLib import dbus import dbus.service import inspect import logging import argparse import pprint import traceback import os from collections import defaultdict from functools import partial # our own packages from ve_utils import exit_on_error, wrap_dbus_value, unwrap_dbus_value, add_name_owner_changed_receiver notfound = object() # For lookups where None is a valid result logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) class SystemBus(dbus.bus.BusConnection): def __new__(cls): return dbus.bus.BusConnection.__new__(cls, dbus.bus.BusConnection.TYPE_SYSTEM) class SessionBus(dbus.bus.BusConnection): def __new__(cls): return dbus.bus.BusConnection.__new__(cls, dbus.bus.BusConnection.TYPE_SESSION) class MonitoredValue(object): def __init__(self, value, text, options): super(MonitoredValue, self).__init__() self.value = value self.text = text self.options = options # For legacy code, allow treating this as a tuple/list def __iter__(self): return iter((self.value, self.text, self.options)) class Service(object): def __init__(self, id, serviceName, deviceInstance): super(Service, self).__init__() self.id = id self.name = serviceName self.paths = {} self._seen = set() self.deviceInstance = deviceInstance # For legacy code, attributes can still be accessed as if keys from a # dictionary. def __setitem__(self, key, value): self.__dict__[key] = value def __getitem__(self, key): return self.__dict__[key] def set_seen(self, path): self._seen.add(path) def seen(self, path): return path in self._seen @property def service_class(self): return '.'.join(self.name.split('.')[:3]) class DbusMonitor(object): ## Constructor def __init__(self, dbusTree, valueChangedCallback=None, deviceAddedCallback=None, deviceRemovedCallback=None, namespace="com.victronenergy", ignoreServices=[]): # valueChangedCallback is the callback that we call when something has changed. # def value_changed_on_dbus(dbusServiceName, dbusPath, options, changes, deviceInstance): # in which changes is a tuple with GetText() and GetValue() self.valueChangedCallback = valueChangedCallback self.deviceAddedCallback = deviceAddedCallback self.deviceRemovedCallback = deviceRemovedCallback self.dbusTree = dbusTree self.ignoreServices = ignoreServices # Lists all tracked services. Stores name, id, device instance, value per path, and whenToLog info # indexed by service name (eg. com.victronenergy.settings). self.servicesByName = {} # Same values as self.servicesByName, but indexed by service id (eg. :1.30) self.servicesById = {} # Keep track of services by class to speed up calls to get_service_list self.servicesByClass = defaultdict(list) # Keep track of any additional watches placed on items self.serviceWatches = defaultdict(list) # For a PC, connect to the SessionBus # For a CCGX, connect to the SystemBus self.dbusConn = SessionBus() if 'DBUS_SESSION_BUS_ADDRESS' in os.environ else SystemBus() # subscribe to NameOwnerChange for bus connect / disconnect events. # NOTE: this is on a different bus then the one above! standardBus = (dbus.SessionBus() if 'DBUS_SESSION_BUS_ADDRESS' in os.environ \ else dbus.SystemBus()) add_name_owner_changed_receiver(standardBus, self.dbus_name_owner_changed) # Subscribe to PropertiesChanged for all services self.dbusConn.add_signal_receiver(self.handler_value_changes, dbus_interface='com.victronenergy.BusItem', signal_name='PropertiesChanged', path_keyword='path', sender_keyword='senderId') # Subscribe to ItemsChanged for all services self.dbusConn.add_signal_receiver(self.handler_item_changes, dbus_interface='com.victronenergy.BusItem', signal_name='ItemsChanged', path='/', sender_keyword='senderId') logger.info('===== Search on dbus for services that we will monitor starting... =====') serviceNames = self.dbusConn.list_names() for serviceName in serviceNames: self.scan_dbus_service(serviceName) logger.info('===== Search on dbus for services that we will monitor finished =====') @staticmethod def make_service(serviceId, serviceName, deviceInstance): """ Override this to use a different kind of service object. """ return Service(serviceId, serviceName, deviceInstance) def make_monitor(self, service, path, value, text, options): """ Override this to do more things with monitoring. """ return MonitoredValue(unwrap_dbus_value(value), unwrap_dbus_value(text), options) def dbus_name_owner_changed(self, name, oldowner, newowner): if not name.startswith("com.victronenergy."): return #decouple, and process in main loop GLib.idle_add(exit_on_error, self._process_name_owner_changed, name, oldowner, newowner) def _process_name_owner_changed(self, name, oldowner, newowner): if newowner != '': # so we found some new service. Check if we can do something with it. newdeviceadded = self.scan_dbus_service(name) if newdeviceadded and self.deviceAddedCallback is not None: self.deviceAddedCallback(name, self.get_device_instance(name)) elif name in self.servicesByName: # it disappeared, we need to remove it. logger.info("%s disappeared from the dbus. Removing it from our lists" % name) service = self.servicesByName[name] del self.servicesById[service.id] del self.servicesByName[name] for watch in self.serviceWatches[name]: watch.remove() del self.serviceWatches[name] self.servicesByClass[service.service_class].remove(service) if self.deviceRemovedCallback is not None: self.deviceRemovedCallback(name, service.deviceInstance) def scan_dbus_service(self, serviceName): try: return self.scan_dbus_service_inner(serviceName) except: logger.error("Ignoring %s because of error while scanning:" % (serviceName)) traceback.print_exc() return False # Errors 'org.freedesktop.DBus.Error.ServiceUnknown' and # 'org.freedesktop.DBus.Error.Disconnected' seem to happen when the service # disappears while its being scanned. Which might happen, but is not really # normal either, so letting them go into the logs. # Scans the given dbus service to see if it contains anything interesting for us. If it does, add # it to our list of monitored D-Bus services. def scan_dbus_service_inner(self, serviceName): # make it a normal string instead of dbus string serviceName = str(serviceName) if (len(self.ignoreServices) != 0 and any(serviceName.startswith(x) for x in self.ignoreServices)): logger.debug("Ignoring service %s" % serviceName) return False paths = self.dbusTree.get('.'.join(serviceName.split('.')[0:3]), None) if paths is None: logger.debug("Ignoring service %s, not in the tree" % serviceName) return False logger.info("Found: %s, scanning and storing items" % serviceName) serviceId = self.dbusConn.get_name_owner(serviceName) # we should never be notified to add a D-Bus service that we already have. If this assertion # raises, check process_name_owner_changed, and D-Bus workings. assert serviceName not in self.servicesByName assert serviceId not in self.servicesById # Try to fetch everything with a GetItems, then fall back to older # methods if that fails try: values = self.dbusConn.call_blocking(serviceName, '/', None, 'GetItems', '', []) except dbus.exceptions.DBusException: logger.info("GetItems failed, trying legacy methods") else: return self.scan_dbus_service_getitems_done(serviceName, serviceId, values) if serviceName == 'com.victronenergy.settings': di = 0 elif serviceName.startswith('com.victronenergy.vecan.'): di = 0 else: try: di = self.dbusConn.call_blocking(serviceName, '/DeviceInstance', None, 'GetValue', '', []) except dbus.exceptions.DBusException: logger.info(" %s was skipped because it has no device instance" % serviceName) return False # Skip it else: di = int(di) logger.info(" %s has device instance %s" % (serviceName, di)) service = self.make_service(serviceId, serviceName, di) # Let's try to fetch everything in one go values = {} texts = {} try: values.update(self.dbusConn.call_blocking(serviceName, '/', None, 'GetValue', '', [])) texts.update(self.dbusConn.call_blocking(serviceName, '/', None, 'GetText', '', [])) except: pass for path, options in paths.items(): # path will be the D-Bus path: '/Ac/ActiveIn/L1/V' # options will be a dictionary: {'code': 'V', 'whenToLog': 'onIntervalAlways'} # Try to obtain the value we want from our bulk fetch. If we # cannot find it there, do an individual query. value = values.get(path[1:], notfound) if value != notfound: service.set_seen(path) text = texts.get(path[1:], notfound) if value is notfound or text is notfound: try: value = self.dbusConn.call_blocking(serviceName, path, None, 'GetValue', '', []) service.set_seen(path) text = self.dbusConn.call_blocking(serviceName, path, None, 'GetText', '', []) except dbus.exceptions.DBusException as e: if e.get_dbus_name() in ( 'org.freedesktop.DBus.Error.ServiceUnknown', 'org.freedesktop.DBus.Error.Disconnected'): raise # This exception will be handled below # TODO org.freedesktop.DBus.Error.UnknownMethod really # shouldn't happen but sometimes does. logger.debug("%s %s does not exist (yet)" % (serviceName, path)) value = None text = None service.paths[path] = self.make_monitor(service, path, unwrap_dbus_value(value), unwrap_dbus_value(text), options) logger.debug("Finished scanning and storing items for %s" % serviceName) # Adjust self at the end of the scan, so we don't have an incomplete set of # data if an exception occurs during the scan. self.servicesByName[serviceName] = service self.servicesById[serviceId] = service self.servicesByClass[service.service_class].append(service) return True def scan_dbus_service_getitems_done(self, serviceName, serviceId, values): # Keeping these exceptions for legacy reasons if serviceName == 'com.victronenergy.settings': di = 0 elif serviceName.startswith('com.victronenergy.vecan.'): di = 0 else: try: di = values['/DeviceInstance']['Value'] except KeyError: logger.info(" %s was skipped because it has no device instance" % serviceName) return False else: di = int(di) logger.info(" %s has device instance %s" % (serviceName, di)) service = self.make_service(serviceId, serviceName, di) paths = self.dbusTree.get('.'.join(serviceName.split('.')[0:3]), {}) for path, options in paths.items(): item = values.get(path, notfound) if item is notfound: service.paths[path] = self.make_monitor(service, path, None, None, options) else: service.set_seen(path) value = item.get('Value', None) text = item.get('Text', None) service.paths[path] = self.make_monitor(service, path, unwrap_dbus_value(value), unwrap_dbus_value(text), options) self.servicesByName[serviceName] = service self.servicesById[serviceId] = service self.servicesByClass[service.service_class].append(service) return True def handler_item_changes(self, items, senderId): if not isinstance(items, dict): return try: service = self.servicesById[senderId] except KeyError: # senderId isn't there, which means it hasn't been scanned yet. return for path, changes in items.items(): try: v = unwrap_dbus_value(changes['Value']) except (KeyError, TypeError): continue try: t = changes['Text'] except KeyError: t = str(v) self._handler_value_changes(service, path, v, t) def handler_value_changes(self, changes, path, senderId): # If this properyChange does not involve a value, our work is done. if 'Value' not in changes: return try: service = self.servicesById[senderId] except KeyError: # senderId isn't there, which means it hasn't been scanned yet. return v = unwrap_dbus_value(changes['Value']) # Some services don't send Text with their PropertiesChanged events. try: t = changes['Text'] except KeyError: t = str(v) self._handler_value_changes(service, path, v, t) def _handler_value_changes(self, service, path, value, text): try: a = service.paths[path] except KeyError: # path isn't there, which means it hasn't been scanned yet. return service.set_seen(path) # First update our store to the new value if a.value == value: return a.value = value a.text = text # And do the rest of the processing in on the mainloop if self.valueChangedCallback is not None: GLib.idle_add(exit_on_error, self._execute_value_changes, service.name, path, { 'Value': value, 'Text': text}, a.options) def _execute_value_changes(self, serviceName, objectPath, changes, options): # double check that the service still exists, as it might have # disappeared between scheduling-for and executing this function. if serviceName not in self.servicesByName: return self.valueChangedCallback(serviceName, objectPath, options, changes, self.get_device_instance(serviceName)) # Gets the value for a certain servicename and path # The default_value is returned when: # 1. When the service doesn't exist. # 2. When the path asked for isn't being monitored. # 3. When the path exists, but has dbus-invalid, ie an empty byte array. # 4. When the path asked for is being monitored, but doesn't exist for that service. def get_value(self, serviceName, objectPath, default_value=None): service = self.servicesByName.get(serviceName, None) if service is None: return default_value value = service.paths.get(objectPath, None) if value is None or value.value is None: return default_value return value.value # returns if a dbus exists now, by doing a blocking dbus call. # Typically seen will be sufficient and doesn't need access to the dbus. def exists(self, serviceName, objectPath): try: self.dbusConn.call_blocking(serviceName, objectPath, None, 'GetValue', '', []) return True except dbus.exceptions.DBusException as e: return False # Returns if there ever was a successful GetValue or valueChanged event. # Unlike get_value this return True also if the actual value is invalid. # # Note: the path might no longer exists anymore, but that doesn't happen in # practice. If a service really wants to reconfigure itself typically it should # reconnect to the dbus which causes it to be rescanned and seen will be updated. # If it is really needed to know if a path still exists, use exists. def seen(self, serviceName, objectPath): try: return self.servicesByName[serviceName].seen(objectPath) except KeyError: return False # Sets the value for a certain servicename and path, returns the return value of the D-Bus SetValue # method. If the underlying item does not exist (the service does not exist, or the objectPath was not # registered) the function will return -1 def set_value(self, serviceName, objectPath, value): # Check if the D-Bus object referenced by serviceName and objectPath is registered. There is no # necessity to do this, but it is in line with previous implementations which kept VeDbusItemImport # objects for registers items only. service = self.servicesByName.get(serviceName, None) if service is None: return -1 if objectPath not in service.paths: return -1 # We do not catch D-Bus exceptions here, because the previous implementation did not do that either. return self.dbusConn.call_blocking(serviceName, objectPath, dbus_interface='com.victronenergy.BusItem', method='SetValue', signature=None, args=[wrap_dbus_value(value)]) # Similar to set_value, but operates asynchronously def set_value_async(self, serviceName, objectPath, value, reply_handler=None, error_handler=None): service = self.servicesByName.get(serviceName, None) if service is not None: if objectPath in service.paths: self.dbusConn.call_async(serviceName, objectPath, dbus_interface='com.victronenergy.BusItem', method='SetValue', signature=None, args=[wrap_dbus_value(value)], reply_handler=reply_handler, error_handler=error_handler) return if error_handler is not None: error_handler(TypeError('Service or path not found, ' 'service=%s, path=%s' % (serviceName, objectPath))) # returns a dictionary, keys are the servicenames, value the instances # optionally use the classfilter to get only a certain type of services, for # example com.victronenergy.battery. def get_service_list(self, classfilter=None): if classfilter is None: return { servicename: service.deviceInstance \ for servicename, service in self.servicesByName.items() } if classfilter not in self.servicesByClass: return {} return { service.name: service.deviceInstance \ for service in self.servicesByClass[classfilter] } def get_device_instance(self, serviceName): return self.servicesByName[serviceName].deviceInstance def track_value(self, serviceName, objectPath, callback, *args, **kwargs): """ A DbusMonitor can watch specific service/path combos for changes so that it is not fully reliant on the global handler_value_changes in this class. Additional watches are deleted automatically when the service disappears from dbus. """ cb = partial(callback, *args, **kwargs) def root_tracker(items): # Check if objectPath in dict try: v = items[objectPath] _v = unwrap_dbus_value(v['Value']) except (KeyError, TypeError): return # not in this dict try: t = v['Text'] except KeyError: cb({'Value': _v }) else: cb({'Value': _v, 'Text': t}) # Track changes on the path, and also on root self.serviceWatches[serviceName].extend(( self.dbusConn.add_signal_receiver(cb, dbus_interface='com.victronenergy.BusItem', signal_name='PropertiesChanged', path=objectPath, bus_name=serviceName), self.dbusConn.add_signal_receiver(root_tracker, dbus_interface='com.victronenergy.BusItem', signal_name='ItemsChanged', path="/", bus_name=serviceName), )) # ====== ALL CODE BELOW THIS LINE IS PURELY FOR DEVELOPING THIS CLASS ====== # Example function that can be used as a starting point to use this code def value_changed_on_dbus(dbusServiceName, dbusPath, dict, changes, deviceInstance): logger.debug("0 ----------------") logger.debug("1 %s%s changed" % (dbusServiceName, dbusPath)) logger.debug("2 vrm dict : %s" % dict) logger.debug("3 changes-text: %s" % changes['Text']) logger.debug("4 changes-value: %s" % changes['Value']) logger.debug("5 deviceInstance: %s" % deviceInstance) logger.debug("6 - end") def nameownerchange(a, b): # used to find memory leaks in dbusmonitor and VeDbusItemImport import gc gc.collect() objects = gc.get_objects() print (len([o for o in objects if type(o).__name__ == 'VeDbusItemImport'])) print (len([o for o in objects if type(o).__name__ == 'SignalMatch'])) print (len(objects)) def print_values(dbusmonitor): a = dbusmonitor.get_value('wrongservice', '/DbusInvalid', default_value=1000) b = dbusmonitor.get_value('com.victronenergy.dummyservice.ttyO1', '/NotInTheMonitorList', default_value=1000) c = dbusmonitor.get_value('com.victronenergy.dummyservice.ttyO1', '/DbusInvalid', default_value=1000) d = dbusmonitor.get_value('com.victronenergy.dummyservice.ttyO1', '/NonExistingButMonitored', default_value=1000) print ("All should be 1000: Wrong Service: %s, NotInTheMonitorList: %s, DbusInvalid: %s, NonExistingButMonitored: %s" % (a, b, c, d)) return True # We have a mainloop, but that is just for developing this code. Normally above class & code is used from # some other class, such as vrmLogger or the pubsub Implementation. def main(): # Init logging logging.basicConfig(level=logging.DEBUG) logger.info(__file__ + " is starting up") # Have a mainloop, so we can send/receive asynchronous calls to and from dbus DBusGMainLoop(set_as_default=True) import os import sys sys.path.insert(1, os.path.join(os.path.dirname(__file__), '../../')) dummy = {'code': None, 'whenToLog': 'configChange', 'accessLevel': None} monitorlist = {'com.victronenergy.dummyservice': { '/Connected': dummy, '/ProductName': dummy, '/Mgmt/Connection': dummy, '/Dc/0/Voltage': dummy, '/Dc/0/Current': dummy, '/Dc/0/Temperature': dummy, '/Load/I': dummy, '/FirmwareVersion': dummy, '/DbusInvalid': dummy, '/NonExistingButMonitored': dummy}} d = DbusMonitor(monitorlist, value_changed_on_dbus, deviceAddedCallback=nameownerchange, deviceRemovedCallback=nameownerchange) GLib.timeout_add(1000, print_values, d) # Start and run the mainloop logger.info("Starting mainloop, responding on only events") mainloop = GLib.MainLoop() mainloop.run() if __name__ == "__main__": main() ================================================ FILE: velib_python/oldestVersion ================================================ v3.40~39 ================================================ FILE: velib_python/settingsdevice.py ================================================ import dbus import logging import time from functools import partial # Local imports from vedbus import VeDbusItemImport ## Indexes for the setting dictonary. PATH = 0 VALUE = 1 MINIMUM = 2 MAXIMUM = 3 SILENT = 4 ## The Settings Device class. # Used by python programs, such as the vrm-logger, to read and write settings they # need to store on disk. And since these settings might be changed from a different # source, such as the GUI, the program can pass an eventCallback that will be called # as soon as some setting is changed. # # The settings are stored in flash via the com.victronenergy.settings service on dbus. # See https://github.com/victronenergy/localsettings for more info. # # If there are settings in de supportSettings list which are not yet on the dbus, # and therefore not yet in the xml file, they will be added through the dbus-addSetting # interface of com.victronenergy.settings. class SettingsDevice(object): ## The constructor processes the tree of dbus-items. # @param bus the system-dbus object # @param name the dbus-service-name of the settings dbus service, 'com.victronenergy.settings' # @param supportedSettings dictionary with all setting-names, and their defaultvalue, min, max and whether # the setting is silent. The 'silent' entry is optional. If set to true, no changes in the setting will # be logged by localsettings. # @param eventCallback function that will be called on changes on any of these settings # @param timeout Maximum interval to wait for localsettings. An exception is thrown at the end of the # interval if the localsettings D-Bus service has not appeared yet. def __init__(self, bus, supportedSettings, eventCallback, name='com.victronenergy.settings', timeout=0): logging.debug("===== Settings device init starting... =====") self._bus = bus self._dbus_name = name self._eventCallback = eventCallback self._values = {} # stored the values, used to pass the old value along on a setting change self._settings = {} count = 0 while True: if 'com.victronenergy.settings' in self._bus.list_names(): break if count == timeout: raise Exception("The settings service com.victronenergy.settings does not exist!") count += 1 logging.info('waiting for settings') time.sleep(1) # Add the items. self.addSettings(supportedSettings) logging.debug("===== Settings device init finished =====") def addSettings(self, settings): for setting, options in settings.items(): silent = len(options) > SILENT and options[SILENT] busitem = self.addSetting(options[PATH], options[VALUE], options[MINIMUM], options[MAXIMUM], silent, callback=partial(self.handleChangedSetting, setting)) self._settings[setting] = busitem self._values[setting] = busitem.get_value() def addSetting(self, path, value, _min, _max, silent=False, callback=None): busitem = VeDbusItemImport(self._bus, self._dbus_name, path, callback) if busitem.exists and (value, _min, _max, silent) == busitem._proxy.GetAttributes(): logging.debug("Setting %s found" % path) else: logging.info("Setting %s does not exist yet or must be adjusted" % path) # Prepare to add the setting. Most dbus types extend the python # type so it is only necessary to additionally test for Int64. if isinstance(value, (int, dbus.Int64)): itemType = 'i' elif isinstance(value, float): itemType = 'f' else: itemType = 's' # Add the setting # TODO, make an object that inherits VeDbusItemImport, and complete the D-Bus settingsitem interface settings_item = VeDbusItemImport(self._bus, self._dbus_name, '/Settings', createsignal=False) setting_path = path.replace('/Settings/', '', 1) if silent: settings_item._proxy.AddSilentSetting('', setting_path, value, itemType, _min, _max) else: settings_item._proxy.AddSetting('', setting_path, value, itemType, _min, _max) busitem = VeDbusItemImport(self._bus, self._dbus_name, path, callback) return busitem def handleChangedSetting(self, setting, servicename, path, changes): oldvalue = self._values[setting] if setting in self._values else None self._values[setting] = changes['Value'] if self._eventCallback is None: return self._eventCallback(setting, oldvalue, changes['Value']) def setDefault(self, path): item = VeDbusItemImport(self._bus, self._dbus_name, path, createsignal=False) item.set_default() def __getitem__(self, setting): return self._settings[setting].get_value() def __setitem__(self, setting, newvalue): result = self._settings[setting].set_value(newvalue) if result != 0: # Trying to make some false change to our own settings? How dumb! assert False ================================================ FILE: velib_python/ve_utils.py ================================================ #!/usr/bin/env python3 # -*- coding: utf-8 -*- import sys from traceback import print_exc from os import _exit as os_exit from os import statvfs from subprocess import check_output, CalledProcessError import logging import dbus logger = logging.getLogger(__name__) VEDBUS_INVALID = dbus.Array([], signature=dbus.Signature('i'), variant_level=1) class NoVrmPortalIdError(Exception): pass # Use this function to make sure the code quits on an unexpected exception. Make sure to use it # when using GLib.idle_add and also GLib.timeout_add. # Without this, the code will just keep running, since GLib does not stop the mainloop on an # exception. # Example: GLib.idle_add(exit_on_error, myfunc, arg1, arg2) def exit_on_error(func, *args, **kwargs): try: return func(*args, **kwargs) except: try: print ('exit_on_error: there was an exception. Printing stacktrace will be tried and then exit') print_exc() except: pass # sys.exit() is not used, since that throws an exception, which does not lead to a program # halt when used in a dbus callback, see connection.py in the Python/Dbus libraries, line 230. os_exit(1) __vrm_portal_id = None def get_vrm_portal_id(): # The original definition of the VRM Portal ID is that it is the mac # address of the onboard- ethernet port (eth0), stripped from its colons # (:) and lower case. This may however differ between platforms. On Venus # the task is therefore deferred to /sbin/get-unique-id so that a # platform specific method can be easily defined. # # If /sbin/get-unique-id does not exist, then use the ethernet address # of eth0. This also handles the case where velib_python is used as a # package install on a Raspberry Pi. # # On a Linux host where the network interface may not be eth0, you can set # the VRM_IFACE environment variable to the correct name. global __vrm_portal_id if __vrm_portal_id: return __vrm_portal_id portal_id = None # First try the method that works if we don't have a data partition. This # will fail when the current user is not root. try: portal_id = check_output("/sbin/get-unique-id").decode("utf-8", "ignore").strip() if not portal_id: raise NoVrmPortalIdError("get-unique-id returned blank") __vrm_portal_id = portal_id return portal_id except CalledProcessError: # get-unique-id returned non-zero raise NoVrmPortalIdError("get-unique-id returned non-zero") except OSError: # File doesn't exist, use fallback pass # Fall back to getting our id using a syscall. Assume we are on linux. # Allow the user to override what interface is used using an environment # variable. import fcntl, socket, struct, os iface = os.environ.get('VRM_IFACE', 'eth0').encode('ascii') s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) try: info = fcntl.ioctl(s.fileno(), 0x8927, struct.pack('256s', iface[:15])) except IOError: raise NoVrmPortalIdError("ioctl failed for eth0") __vrm_portal_id = info[18:24].hex() return __vrm_portal_id # See VE.Can registers - public.docx for definition of this conversion def convert_vreg_version_to_readable(version): def str_to_arr(x, length): a = [] for i in range(0, len(x), length): a.append(x[i:i+length]) return a x = "%x" % version x = x.upper() if len(x) == 5 or len(x) == 3 or len(x) == 1: x = '0' + x a = str_to_arr(x, 2); # remove the first 00 if there are three bytes and it is 00 if len(a) == 3 and a[0] == '00': a.remove(0); # if we have two or three bytes now, and the first character is a 0, remove it if len(a) >= 2 and a[0][0:1] == '0': a[0] = a[0][1]; result = '' for item in a: result += ('.' if result != '' else '') + item result = 'v' + result return result def get_free_space(path): result = -1 try: s = statvfs(path) result = s.f_frsize * s.f_bavail # Number of free bytes that ordinary users except Exception as ex: logger.info("Error while retrieving free space for path %s: %s" % (path, ex)) return result def _get_sysfs_machine_name(): try: with open('/sys/firmware/devicetree/base/model', 'r') as f: return f.read().rstrip('\x00') except IOError: pass return None # Returns None if it cannot find a machine name. Otherwise returns the string # containing the name def get_machine_name(): # First try calling the venus utility script try: return check_output("/usr/bin/product-name").strip().decode('UTF-8') except (CalledProcessError, OSError): pass # Fall back to sysfs name = _get_sysfs_machine_name() if name is not None: return name # Fall back to venus build machine name try: with open('/etc/venus/machine', 'r', encoding='UTF-8') as f: return f.read().strip() except IOError: pass return None def get_product_id(): """ Find the machine ID and return it. """ # First try calling the venus utility script try: return check_output("/usr/bin/product-id").strip().decode('UTF-8') except (CalledProcessError, OSError): pass # Fall back machine name mechanism name = _get_sysfs_machine_name() return { 'Color Control GX': 'C001', 'Venus GX': 'C002', 'Octo GX': 'C006', 'EasySolar-II': 'C007', 'MultiPlus-II': 'C008', 'Maxi GX': 'C009', 'Cerbo GX': 'C00A' }.get(name, 'C003') # C003 is Generic # Returns False if it cannot open the file. Otherwise returns its rstripped contents def read_file(path): content = False try: with open(path, 'r') as f: content = f.read().rstrip() except Exception as ex: logger.debug("Error while reading %s: %s" % (path, ex)) return content def wrap_dbus_value(value): if value is None: return VEDBUS_INVALID if isinstance(value, float): return dbus.Double(value, variant_level=1) if isinstance(value, bool): return dbus.Boolean(value, variant_level=1) if isinstance(value, int): try: return dbus.Int32(value, variant_level=1) except OverflowError: return dbus.Int64(value, variant_level=1) if isinstance(value, str): return dbus.String(value, variant_level=1) if isinstance(value, list): if len(value) == 0: # If the list is empty we cannot infer the type of the contents. So assume unsigned integer. # A (signed) integer is dangerous, because an empty list of signed integers is used to encode # an invalid value. return dbus.Array([], signature=dbus.Signature('u'), variant_level=1) return dbus.Array([wrap_dbus_value(x) for x in value], variant_level=1) if isinstance(value, dict): # Wrapping the keys of the dictionary causes D-Bus errors like: # 'arguments to dbus_message_iter_open_container() were incorrect, # assertion "(type == DBUS_TYPE_ARRAY && contained_signature && # *contained_signature == DBUS_DICT_ENTRY_BEGIN_CHAR) || (contained_signature == NULL || # _dbus_check_is_valid_signature (contained_signature))" failed in file ...' return dbus.Dictionary({(k, wrap_dbus_value(v)) for k, v in value.items()}, variant_level=1) return value dbus_int_types = (dbus.Int32, dbus.UInt32, dbus.Byte, dbus.Int16, dbus.UInt16, dbus.UInt32, dbus.Int64, dbus.UInt64) def unwrap_dbus_value(val): """Converts D-Bus values back to the original type. For example if val is of type DBus.Double, a float will be returned.""" if isinstance(val, dbus_int_types): return int(val) if isinstance(val, dbus.Double): return float(val) if isinstance(val, dbus.Array): v = [unwrap_dbus_value(x) for x in val] return None if len(v) == 0 else v if isinstance(val, (dbus.Signature, dbus.String)): return str(val) # Python has no byte type, so we convert to an integer. if isinstance(val, dbus.Byte): return int(val) if isinstance(val, dbus.ByteArray): return "".join([bytes(x) for x in val]) if isinstance(val, (list, tuple)): return [unwrap_dbus_value(x) for x in val] if isinstance(val, (dbus.Dictionary, dict)): # Do not unwrap the keys, see comment in wrap_dbus_value return dict([(x, unwrap_dbus_value(y)) for x, y in val.items()]) if isinstance(val, dbus.Boolean): return bool(val) return val # When supported, only name owner changes for the the given namespace are reported. This # prevents spending cpu time at irrelevant changes, like scripts accessing the bus temporarily. def add_name_owner_changed_receiver(dbus, name_owner_changed, namespace="com.victronenergy"): # support for arg0namespace is submitted upstream, but not included at the time of # writing, Venus OS does support it, so try if it works. if namespace is None: dbus.add_signal_receiver(name_owner_changed, signal_name='NameOwnerChanged') else: try: dbus.add_signal_receiver(name_owner_changed, signal_name='NameOwnerChanged', arg0namespace=namespace) except TypeError: dbus.add_signal_receiver(name_owner_changed, signal_name='NameOwnerChanged') ================================================ FILE: velib_python/vedbus.py ================================================ #!/usr/bin/env python3 # -*- coding: utf-8 -*- import dbus.service import logging import traceback import os import weakref from collections import defaultdict from ve_utils import wrap_dbus_value, unwrap_dbus_value # vedbus contains three classes: # VeDbusItemImport -> use this to read data from the dbus, ie import # VeDbusItemExport -> use this to export data to the dbus (one value) # VeDbusService -> use that to create a service and export several values to the dbus # Code for VeDbusItemImport is copied from busitem.py and thereafter modified. # All projects that used busitem.py need to migrate to this package. And some # projects used to define there own equivalent of VeDbusItemExport. Better to # use VeDbusItemExport, or even better the VeDbusService class that does it all for you. # TODOS # 1 check for datatypes, it works now, but not sure if all is compliant with # com.victronenergy.BusItem interface definition. See also the files in # tests_and_examples. And see 'if type(v) == dbus.Byte:' on line 102. Perhaps # something similar should also be done in VeDbusBusItemExport? # 2 Shouldn't VeDbusBusItemExport inherit dbus.service.Object? # 7 Make hard rules for services exporting data to the D-Bus, in order to make tracking # changes possible. Does everybody first invalidate its data before leaving the bus? # And what about before taking one object away from the bus, instead of taking the # whole service offline? # They should! And after taking one value away, do we need to know that someone left # the bus? Or we just keep that value in invalidated for ever? Result is that we can't # see the difference anymore between an invalidated value and a value that was first on # the bus and later not anymore. See comments above VeDbusItemImport as well. # 9 there are probably more todos in the code below. # Some thoughts with regards to the data types: # # Text from: http://dbus.freedesktop.org/doc/dbus-python/doc/tutorial.html#data-types # --- # Variants are represented by setting the variant_level keyword argument in the # constructor of any D-Bus data type to a value greater than 0 (variant_level 1 # means a variant containing some other data type, variant_level 2 means a variant # containing a variant containing some other data type, and so on). If a non-variant # is passed as an argument but introspection indicates that a variant is expected, # it'll automatically be wrapped in a variant. # --- # # Also the different dbus datatypes, such as dbus.Int32, and dbus.UInt32 are a subclass # of Python int. dbus.String is a subclass of Python standard class unicode, etcetera # # So all together that explains why we don't need to explicitly convert back and forth # between the dbus datatypes and the standard python datatypes. Note that all datatypes # in python are objects. Even an int is an object. # The signature of a variant is 'v'. # Export ourselves as a D-Bus service. class VeDbusService(object): def __init__(self, servicename, bus=None, register=True): # dict containing the VeDbusItemExport objects, with their path as the key. self._dbusobjects = {} self._dbusnodes = {} self._ratelimiters = [] self._dbusname = None self.name = servicename # dict containing the onchange callbacks, for each object. Object path is the key self._onchangecallbacks = {} # Connect to session bus whenever present, else use the system bus self._dbusconn = bus or (dbus.SessionBus() if 'DBUS_SESSION_BUS_ADDRESS' in os.environ else dbus.SystemBus()) # make the dbus connection available to outside, could make this a true property instead, but ach.. self.dbusconn = self._dbusconn # Add the root item that will return all items as a tree self._dbusnodes['/'] = VeDbusRootExport(self._dbusconn, '/', self) # Immediately register the service unless requested not to if register: self.register() def register(self): # Register ourselves on the dbus, trigger an error if already in use (do_not_queue) self._dbusname = dbus.service.BusName(self.name, self._dbusconn, do_not_queue=True) logging.info("registered ourselves on D-Bus as %s" % self.name) # To force immediate deregistering of this dbus service and all its object paths, explicitly # call __del__(). def __del__(self): for node in list(self._dbusnodes.values()): node.__del__() self._dbusnodes.clear() for item in list(self._dbusobjects.values()): item.__del__() self._dbusobjects.clear() if self._dbusname: self._dbusname.__del__() # Forces call to self._bus.release_name(self._name), see source code self._dbusname = None def get_name(self): return self._dbusname.get_name() # @param callbackonchange function that will be called when this value is changed. First parameter will # be the path of the object, second the new value. This callback should return # True to accept the change, False to reject it. def add_path(self, path, value, description="", writeable=False, onchangecallback=None, gettextcallback=None, valuetype=None, itemtype=None): if onchangecallback is not None: self._onchangecallbacks[path] = onchangecallback itemtype = itemtype or VeDbusItemExport item = itemtype(self._dbusconn, path, value, description, writeable, self._value_changed, gettextcallback, deletecallback=self._item_deleted, valuetype=valuetype) spl = path.split('/') for i in range(2, len(spl)): subPath = '/'.join(spl[:i]) if subPath not in self._dbusnodes and subPath not in self._dbusobjects: self._dbusnodes[subPath] = VeDbusTreeExport(self._dbusconn, subPath, self) self._dbusobjects[path] = item logging.debug('added %s with start value %s. Writeable is %s' % (path, value, writeable)) return item # Add the mandatory paths, as per victron dbus api doc def add_mandatory_paths(self, processname, processversion, connection, deviceinstance, productid, productname, firmwareversion, hardwareversion, connected): self.add_path('/Mgmt/ProcessName', processname) self.add_path('/Mgmt/ProcessVersion', processversion) self.add_path('/Mgmt/Connection', connection) # Create rest of the mandatory objects self.add_path('/DeviceInstance', deviceinstance) self.add_path('/ProductId', productid) self.add_path('/ProductName', productname) self.add_path('/FirmwareVersion', firmwareversion) self.add_path('/HardwareVersion', hardwareversion) self.add_path('/Connected', connected) # Callback function that is called from the VeDbusItemExport objects when a value changes. This function # maps the change-request to the onchangecallback given to us for this specific path. def _value_changed(self, path, newvalue): if path not in self._onchangecallbacks: return True return self._onchangecallbacks[path](path, newvalue) def _item_deleted(self, path): self._dbusobjects.pop(path) for np in list(self._dbusnodes.keys()): if np != '/': for ip in self._dbusobjects: if ip.startswith(np + '/'): break else: self._dbusnodes[np].__del__() self._dbusnodes.pop(np) def __getitem__(self, path): return self._dbusobjects[path].local_get_value() def __setitem__(self, path, newvalue): self._dbusobjects[path].local_set_value(newvalue) def __delitem__(self, path): self._dbusobjects[path].__del__() # Invalidates and then removes the object path assert path not in self._dbusobjects def __contains__(self, path): return path in self._dbusobjects def __enter__(self): l = ServiceContext(self) self._ratelimiters.append(l) return l def __exit__(self, *exc): # pop off the top one and flush it. If with statements are nested # then each exit flushes its own part. if self._ratelimiters: self._ratelimiters.pop().flush() class ServiceContext(object): def __init__(self, parent): self.parent = parent self.changes = {} def __contains__(self, path): return path in self.parent def __getitem__(self, path): return self.parent[path] def __setitem__(self, path, newvalue): c = self.parent._dbusobjects[path]._local_set_value(newvalue) if c is not None: self.changes[path] = c def __delitem__(self, path): if path in self.changes: del self.changes[path] del self.parent[path] def flush(self): if self.changes: self.parent._dbusnodes['/'].ItemsChanged(self.changes) self.changes.clear() def add_path(self, path, value, *args, **kwargs): self.parent.add_path(path, value, *args, **kwargs) self.changes[path] = { 'Value': wrap_dbus_value(value), 'Text': self.parent._dbusobjects[path].GetText() } def del_tree(self, root): root = root.rstrip('/') for p in list(self.parent._dbusobjects.keys()): if p == root or p.startswith(root + '/'): self[p] = None self.parent._dbusobjects[p].__del__() def get_name(self): return self.parent.get_name() class TrackerDict(defaultdict): """ Same as defaultdict, but passes the key to default_factory. """ def __missing__(self, key): self[key] = x = self.default_factory(key) return x class VeDbusRootTracker(object): """ This tracks the root of a dbus path and listens for PropertiesChanged signals. When a signal arrives, parse it and unpack the key/value changes into traditional events, then pass it to the original eventCallback method. """ def __init__(self, bus, serviceName): self.importers = defaultdict(weakref.WeakSet) self.serviceName = serviceName self._match = bus.get_object(serviceName, '/', introspect=False).connect_to_signal( "ItemsChanged", weak_functor(self._items_changed_handler)) def __del__(self): self._match.remove() self._match = None def add(self, i): self.importers[i.path].add(i) def _items_changed_handler(self, items): if not isinstance(items, dict): return for path, changes in items.items(): try: v = changes['Value'] except KeyError: continue try: t = changes['Text'] except KeyError: t = str(unwrap_dbus_value(v)) for i in self.importers.get(path, ()): i._properties_changed_handler({'Value': v, 'Text': t}) """ Importing basics: - If when we power up, the D-Bus service does not exist, or it does exist and the path does not yet exist, still subscribe to a signal: as soon as it comes online it will send a signal with its initial value, which VeDbusItemImport will receive and use to update local cache. And, when set, call the eventCallback. - If when we power up, save it - When using get_value, know that there is no difference between services (or object paths) that don't exist and paths that are invalid (= empty array, see above). Both will return None. In case you do really want to know ifa path exists or not, use the exists property. - When a D-Bus service leaves the D-Bus, it will first invalidate all its values, and send signals with that update, and only then leave the D-Bus. (or do we need to subscribe to the NameOwnerChanged- signal!?!) To be discussed and make sure. Not really urgent, since all existing code that uses this class already subscribes to the NameOwnerChanged signal, and subsequently removes instances of this class. Read when using this class: Note that when a service leaves that D-Bus without invalidating all its exported objects first, for example because it is killed, VeDbusItemImport doesn't have a clue. So when using VeDbusItemImport, make sure to also subscribe to the NamerOwnerChanged signal on bus-level. Or just use dbusmonitor, because that takes care of all of that for you. """ class VeDbusItemImport(object): def __new__(cls, bus, serviceName, path, eventCallback=None, createsignal=True): instance = object.__new__(cls) # If signal tracking should be done, also add to root tracker if createsignal: if "_roots" not in cls.__dict__: cls._roots = TrackerDict(lambda k: VeDbusRootTracker(bus, k)) return instance ## Constructor # @param bus the bus-object (SESSION or SYSTEM). # @param serviceName the dbus-service-name (string), for example 'com.victronenergy.battery.ttyO1' # @param path the object-path, for example '/Dc/V' # @param eventCallback function that you want to be called on a value change # @param createSignal only set this to False if you use this function to one time read a value. When # leaving it to True, make sure to also subscribe to the NameOwnerChanged signal # elsewhere. See also note some 15 lines up. def __init__(self, bus, serviceName, path, eventCallback=None, createsignal=True): # TODO: is it necessary to store _serviceName and _path? Isn't it # stored in the bus_getobjectsomewhere? self._serviceName = serviceName self._path = path self._match = None # TODO: _proxy is being used in settingsdevice.py, make a getter for that self._proxy = bus.get_object(serviceName, path, introspect=False) self.eventCallback = eventCallback assert eventCallback is None or createsignal == True if createsignal: self._match = self._proxy.connect_to_signal( "PropertiesChanged", weak_functor(self._properties_changed_handler)) self._roots[serviceName].add(self) # store the current value in _cachedvalue. When it doesn't exists set _cachedvalue to # None, same as when a value is invalid self._cachedvalue = None try: v = self._proxy.GetValue() except dbus.exceptions.DBusException: pass else: self._cachedvalue = unwrap_dbus_value(v) def __del__(self): if self._match is not None: self._match.remove() self._match = None self._proxy = None def _refreshcachedvalue(self): self._cachedvalue = unwrap_dbus_value(self._proxy.GetValue()) ## Returns the path as a string, for example '/AC/L1/V' @property def path(self): return self._path ## Returns the dbus service name as a string, for example com.victronenergy.vebus.ttyO1 @property def serviceName(self): return self._serviceName ## Returns the value of the dbus-item. # the type will be a dbus variant, for example dbus.Int32(0, variant_level=1) # this is not a property to keep the name consistant with the com.victronenergy.busitem interface # returns None when the property is invalid def get_value(self): return self._cachedvalue ## Writes a new value to the dbus-item def set_value(self, newvalue): r = self._proxy.SetValue(wrap_dbus_value(newvalue)) # instead of just saving the value, go to the dbus and get it. So we have the right type etc. if r == 0: self._refreshcachedvalue() return r ## Resets the item to its default value def set_default(self): self._proxy.SetDefault() self._refreshcachedvalue() ## Returns the text representation of the value. # For example when the value is an enum/int GetText might return the string # belonging to that enum value. Another example, for a voltage, GetValue # would return a float, 12.0Volt, and GetText could return 12 VDC. # # Note that this depends on how the dbus-producer has implemented this. def get_text(self): return self._proxy.GetText() ## Returns true of object path exists, and false if it doesn't @property def exists(self): # TODO: do some real check instead of this crazy thing. r = False try: r = self._proxy.GetValue() r = True except dbus.exceptions.DBusException: pass return r ## callback for the trigger-event. # @param eventCallback the event-callback-function. @property def eventCallback(self): return self._eventCallback @eventCallback.setter def eventCallback(self, eventCallback): self._eventCallback = eventCallback ## Is called when the value of the imported bus-item changes. # Stores the new value in our local cache, and calls the eventCallback, if set. def _properties_changed_handler(self, changes): if "Value" in changes: changes['Value'] = unwrap_dbus_value(changes['Value']) self._cachedvalue = changes['Value'] if self._eventCallback: # The reason behind this try/except is to prevent errors silently ending up the an error # handler in the dbus code. try: self._eventCallback(self._serviceName, self._path, changes) except: traceback.print_exc() os._exit(1) # sys.exit() is not used, since that also throws an exception class VeDbusTreeExport(dbus.service.Object): def __init__(self, bus, objectPath, service): dbus.service.Object.__init__(self, bus, objectPath) self._service = service logging.debug("VeDbusTreeExport %s has been created" % objectPath) def __del__(self): # self._get_path() will raise an exception when retrieved after the call to .remove_from_connection, # so we need a copy. path = self._get_path() if path is None: return self.remove_from_connection() logging.debug("VeDbusTreeExport %s has been removed" % path) def _get_path(self): if len(self._locations) == 0: return None return self._locations[0][1] def _get_value_handler(self, path, get_text=False): logging.debug("_get_value_handler called for %s" % path) r = {} px = path if not px.endswith('/'): px += '/' for p, item in self._service._dbusobjects.items(): if p.startswith(px): v = item.GetText() if get_text else wrap_dbus_value(item.local_get_value()) r[p[len(px):]] = v logging.debug(r) return r @dbus.service.method('com.victronenergy.BusItem', out_signature='v') def GetValue(self): value = self._get_value_handler(self._get_path()) return dbus.Dictionary(value, signature=dbus.Signature('sv'), variant_level=1) @dbus.service.method('com.victronenergy.BusItem', out_signature='v') def GetText(self): return self._get_value_handler(self._get_path(), True) def local_get_value(self): return self._get_value_handler(self.path) class VeDbusRootExport(VeDbusTreeExport): @dbus.service.signal('com.victronenergy.BusItem', signature='a{sa{sv}}') def ItemsChanged(self, changes): pass @dbus.service.method('com.victronenergy.BusItem', out_signature='a{sa{sv}}') def GetItems(self): return { path: { 'Value': wrap_dbus_value(item.local_get_value()), 'Text': item.GetText() } for path, item in self._service._dbusobjects.items() } class VeDbusItemExport(dbus.service.Object): ## Constructor of VeDbusItemExport # # Use this object to export (publish), values on the dbus # Creates the dbus-object under the given dbus-service-name. # @param bus The dbus object. # @param objectPath The dbus-object-path. # @param value Value to initialize ourselves with, defaults to None which means Invalid # @param description String containing a description. Can be called over the dbus with GetDescription() # @param writeable what would this do!? :). # @param callback Function that will be called when someone else changes the value of this VeBusItem # over the dbus. First parameter passed to callback will be our path, second the new # value. This callback should return True to accept the change, False to reject it. def __init__(self, bus, objectPath, value=None, description=None, writeable=False, onchangecallback=None, gettextcallback=None, deletecallback=None, valuetype=None): dbus.service.Object.__init__(self, bus, objectPath) self._onchangecallback = onchangecallback self._gettextcallback = gettextcallback self._value = value self._description = description self._writeable = writeable self._deletecallback = deletecallback self._type = valuetype # To force immediate deregistering of this dbus object, explicitly call __del__(). def __del__(self): # self._get_path() will raise an exception when retrieved after the # call to .remove_from_connection, so we need a copy. path = self._get_path() if path == None: return if self._deletecallback is not None: self._deletecallback(path) self.remove_from_connection() logging.debug("VeDbusItemExport %s has been removed" % path) def _get_path(self): if len(self._locations) == 0: return None return self._locations[0][1] ## Sets the value. And in case the value is different from what it was, a signal # will be emitted to the dbus. This function is to be used in the python code that # is using this class to export values to the dbus. # set value to None to indicate that it is Invalid def local_set_value(self, newvalue): changes = self._local_set_value(newvalue) if changes is not None: self.PropertiesChanged(changes) def _local_set_value(self, newvalue): if self._value == newvalue: return None self._value = newvalue return { 'Value': wrap_dbus_value(newvalue), 'Text': self.GetText() } def local_get_value(self): return self._value # ==== ALL FUNCTIONS BELOW THIS LINE WILL BE CALLED BY OTHER PROCESSES OVER THE DBUS ==== ## Dbus exported method SetValue # Function is called over the D-Bus by other process. It will first check (via callback) if new # value is accepted. And it is, stores it and emits a changed-signal. # @param value The new value. # @return completion-code When successful a 0 is return, and when not a -1 is returned. @dbus.service.method('com.victronenergy.BusItem', in_signature='v', out_signature='i') def SetValue(self, newvalue): if not self._writeable: return 1 # NOT OK newvalue = unwrap_dbus_value(newvalue) # If value type is enforced, cast it. If the type can be coerced # python will do it for us. This allows ints to become floats, # or bools to become ints. Additionally also allow None, so that # a path may be invalidated. if self._type is not None and newvalue is not None: try: newvalue = self._type(newvalue) except (ValueError, TypeError): return 1 # NOT OK if newvalue == self._value: return 0 # OK # call the callback given to us, and check if new value is OK. if (self._onchangecallback is None or (self._onchangecallback is not None and self._onchangecallback(self.__dbus_object_path__, newvalue))): self.local_set_value(newvalue) return 0 # OK return 2 # NOT OK ## Dbus exported method GetDescription # # Returns the a description. # @param language A language code (e.g. ISO 639-1 en-US). # @param length Lenght of the language string. # @return description @dbus.service.method('com.victronenergy.BusItem', in_signature='si', out_signature='s') def GetDescription(self, language, length): return self._description if self._description is not None else 'No description given' ## Dbus exported method GetValue # Returns the value. # @return the value when valid, and otherwise an empty array @dbus.service.method('com.victronenergy.BusItem', out_signature='v') def GetValue(self): return wrap_dbus_value(self._value) ## Dbus exported method GetText # Returns the value as string of the dbus-object-path. # @return text A text-value. '---' when local value is invalid @dbus.service.method('com.victronenergy.BusItem', out_signature='s') def GetText(self): if self._value is None: return '---' # Default conversion from dbus.Byte will get you a character (so 'T' instead of '84'), so we # have to convert to int first. Note that if a dbus.Byte turns up here, it must have come from # the application itself, as all data from the D-Bus should have been unwrapped by now. if self._gettextcallback is None and type(self._value) == dbus.Byte: return str(int(self._value)) if self._gettextcallback is None and self.__dbus_object_path__ == '/ProductId': return "0x%X" % self._value if self._gettextcallback is None: return str(self._value) return self._gettextcallback(self.__dbus_object_path__, self._value) ## The signal that indicates that the value has changed. # Other processes connected to this BusItem object will have subscribed to the # event when they want to track our state. @dbus.service.signal('com.victronenergy.BusItem', signature='a{sv}') def PropertiesChanged(self, changes): pass ## This class behaves like a regular reference to a class method (eg. self.foo), but keeps a weak reference ## to the object which method is to be called. ## Use this object to break circular references. class weak_functor: def __init__(self, f): self._r = weakref.ref(f.__self__) self._f = weakref.ref(f.__func__) def __call__(self, *args, **kargs): r = self._r() f = self._f() if r == None or f == None: return f(r, *args, **kargs) ================================================ FILE: velib_python/velib_python/latest/dbusmonitor.py ================================================ #!/usr/bin/env python3 # -*- coding: utf-8 -*- ## @package dbus_vrm # This code takes care of the D-Bus interface (not all of below is implemented yet): # - on startup it scans the dbus for services we know. For each known service found, it searches for # objects/paths we know. Everything we find is stored in items{}, and an event is registered: if a # value changes weĺl be notified and can pass that on to our owner. For example the vrmLogger. # we know. # - after startup, it continues to monitor the dbus: # 1) when services are added we do the same check on that # 2) when services are removed, we remove any items that we had that referred to that service # 3) if an existing services adds paths we update ourselves as well: on init, we make a # VeDbusItemImport for a non-, or not yet existing objectpaths as well1 # # Code is used by the vrmLogger, and also the pubsub code. Both are other modules in the dbus_vrm repo. from dbus.mainloop.glib import DBusGMainLoop from gi.repository import GLib import dbus import dbus.service import inspect import logging import argparse import pprint import traceback import os from collections import defaultdict from functools import partial # our own packages from ve_utils import exit_on_error, wrap_dbus_value, unwrap_dbus_value, add_name_owner_changed_receiver # dbus interface VE_INTERFACE = "com.victronenergy.BusItem" # For lookups where None is a valid result notfound = object() logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) class SystemBus(dbus.bus.BusConnection): def __new__(cls): return dbus.bus.BusConnection.__new__(cls, dbus.bus.BusConnection.TYPE_SYSTEM) class SessionBus(dbus.bus.BusConnection): def __new__(cls): return dbus.bus.BusConnection.__new__(cls, dbus.bus.BusConnection.TYPE_SESSION) class MonitoredValue(object): def __init__(self, value, text, options): super(MonitoredValue, self).__init__() self.value = value self.text = text self.options = options # For legacy code, allow treating this as a tuple/list def __iter__(self): return iter((self.value, self.text, self.options)) class Service(object): def __init__(self, id, serviceName, deviceInstance): super(Service, self).__init__() self.id = id self.name = serviceName self.paths = {} self._seen = set() self.deviceInstance = deviceInstance # For legacy code, attributes can still be accessed as if keys from a # dictionary. def __setitem__(self, key, value): self.__dict__[key] = value def __getitem__(self, key): return self.__dict__[key] def set_seen(self, path): self._seen.add(path) def seen(self, path): return path in self._seen @property def service_class(self): return '.'.join(self.name.split('.')[:3]) class DbusMonitor(object): ## Constructor def __init__(self, dbusTree, valueChangedCallback=None, deviceAddedCallback=None, deviceRemovedCallback=None, namespace="com.victronenergy", ignoreServices=[]): # valueChangedCallback is the callback that we call when something has changed. # def value_changed_on_dbus(dbusServiceName, dbusPath, options, changes, deviceInstance): # in which changes is a tuple with GetText() and GetValue() self.valueChangedCallback = valueChangedCallback self.deviceAddedCallback = deviceAddedCallback self.deviceRemovedCallback = deviceRemovedCallback self.dbusTree = dbusTree self.ignoreServices = ignoreServices # Lists all tracked services. Stores name, id, device instance, value per path, and whenToLog info # indexed by service name (eg. com.victronenergy.settings). self.servicesByName = {} # Same values as self.servicesByName, but indexed by service id (eg. :1.30) self.servicesById = {} # Keep track of services by class to speed up calls to get_service_list self.servicesByClass = defaultdict(list) # Keep track of any additional watches placed on items self.serviceWatches = defaultdict(list) # For a PC, connect to the SessionBus # For a CCGX, connect to the SystemBus self.dbusConn = SessionBus() if 'DBUS_SESSION_BUS_ADDRESS' in os.environ else SystemBus() # subscribe to NameOwnerChange for bus connect / disconnect events. # NOTE: this is on a different bus then the one above! standardBus = (dbus.SessionBus() if 'DBUS_SESSION_BUS_ADDRESS' in os.environ \ else dbus.SystemBus()) add_name_owner_changed_receiver(standardBus, self.dbus_name_owner_changed) # Subscribe to PropertiesChanged for all services self.dbusConn.add_signal_receiver(self.handler_value_changes, dbus_interface='com.victronenergy.BusItem', signal_name='PropertiesChanged', path_keyword='path', sender_keyword='senderId') # Subscribe to ItemsChanged for all services self.dbusConn.add_signal_receiver(self.handler_item_changes, dbus_interface='com.victronenergy.BusItem', signal_name='ItemsChanged', path='/', sender_keyword='senderId') logger.info('===== Search on dbus for services that we will monitor starting... =====') serviceNames = self.dbusConn.list_names() for serviceName in serviceNames: self.scan_dbus_service(serviceName) logger.info('===== Search on dbus for services that we will monitor finished =====') @staticmethod def make_service(serviceId, serviceName, deviceInstance): """ Override this to use a different kind of service object. """ return Service(serviceId, serviceName, deviceInstance) def make_monitor(self, service, path, value, text, options): """ Override this to do more things with monitoring. """ return MonitoredValue(unwrap_dbus_value(value), unwrap_dbus_value(text), options) def dbus_name_owner_changed(self, name, oldowner, newowner): if not name.startswith("com.victronenergy."): return #decouple, and process in main loop GLib.idle_add(exit_on_error, self._process_name_owner_changed, name, oldowner, newowner) def _process_name_owner_changed(self, name, oldowner, newowner): if newowner != '': # so we found some new service. Check if we can do something with it. newdeviceadded = self.scan_dbus_service(name) if newdeviceadded and self.deviceAddedCallback is not None: self.deviceAddedCallback(name, self.get_device_instance(name)) elif name in self.servicesByName: # it disappeared, we need to remove it. logger.info("%s disappeared from the dbus. Removing it from our lists" % name) service = self.servicesByName[name] del self.servicesById[service.id] del self.servicesByName[name] for watch in self.serviceWatches[name]: watch.remove() del self.serviceWatches[name] self.servicesByClass[service.service_class].remove(service) if self.deviceRemovedCallback is not None: self.deviceRemovedCallback(name, service.deviceInstance) def scan_dbus_service(self, serviceName): try: return self.scan_dbus_service_inner(serviceName) except: logger.error("Ignoring %s because of error while scanning:" % (serviceName)) traceback.print_exc() return False # Errors 'org.freedesktop.DBus.Error.ServiceUnknown' and # 'org.freedesktop.DBus.Error.Disconnected' seem to happen when the service # disappears while its being scanned. Which might happen, but is not really # normal either, so letting them go into the logs. # Scans the given dbus service to see if it contains anything interesting for us. If it does, add # it to our list of monitored D-Bus services. def scan_dbus_service_inner(self, serviceName): # make it a normal string instead of dbus string serviceName = str(serviceName) if (len(self.ignoreServices) != 0 and any(serviceName.startswith(x) for x in self.ignoreServices)): logger.debug("Ignoring service %s" % serviceName) return False paths = self.dbusTree.get('.'.join(serviceName.split('.')[0:3]), None) if paths is None: logger.debug("Ignoring service %s, not in the tree" % serviceName) return False logger.info("Found: %s, scanning and storing items" % serviceName) serviceId = self.dbusConn.get_name_owner(serviceName) # we should never be notified to add a D-Bus service that we already have. If this assertion # raises, check process_name_owner_changed, and D-Bus workings. assert serviceName not in self.servicesByName assert serviceId not in self.servicesById # Try to fetch everything with a GetItems, then fall back to older # methods if that fails try: values = self.dbusConn.call_blocking(serviceName, '/', VE_INTERFACE, 'GetItems', '', []) except dbus.exceptions.DBusException: logger.info("GetItems failed, trying legacy methods") else: return self.scan_dbus_service_getitems_done(serviceName, serviceId, values) if serviceName == 'com.victronenergy.settings': di = 0 elif serviceName.startswith('com.victronenergy.vecan.'): di = 0 else: try: di = self.dbusConn.call_blocking(serviceName, '/DeviceInstance', VE_INTERFACE, 'GetValue', '', []) except dbus.exceptions.DBusException: logger.info(" %s was skipped because it has no device instance" % serviceName) return False # Skip it else: di = int(di) logger.info(" %s has device instance %s" % (serviceName, di)) service = self.make_service(serviceId, serviceName, di) # Let's try to fetch everything in one go values = {} texts = {} try: values.update(self.dbusConn.call_blocking(serviceName, '/', VE_INTERFACE, 'GetValue', '', [])) texts.update(self.dbusConn.call_blocking(serviceName, '/', VE_INTERFACE, 'GetText', '', [])) except: pass for path, options in paths.items(): # path will be the D-Bus path: '/Ac/ActiveIn/L1/V' # options will be a dictionary: {'code': 'V', 'whenToLog': 'onIntervalAlways'} # Try to obtain the value we want from our bulk fetch. If we # cannot find it there, do an individual query. value = values.get(path[1:], notfound) if value != notfound: service.set_seen(path) text = texts.get(path[1:], notfound) if value is notfound or text is notfound: try: value = self.dbusConn.call_blocking(serviceName, path, VE_INTERFACE, 'GetValue', '', []) service.set_seen(path) text = self.dbusConn.call_blocking(serviceName, path, VE_INTERFACE, 'GetText', '', []) except dbus.exceptions.DBusException as e: if e.get_dbus_name() in ( 'org.freedesktop.DBus.Error.ServiceUnknown', 'org.freedesktop.DBus.Error.Disconnected'): raise # This exception will be handled below # TODO org.freedesktop.DBus.Error.UnknownMethod really # shouldn't happen but sometimes does. logger.debug("%s %s does not exist (yet)" % (serviceName, path)) value = None text = None service.paths[path] = self.make_monitor(service, path, unwrap_dbus_value(value), unwrap_dbus_value(text), options) logger.debug("Finished scanning and storing items for %s" % serviceName) # Adjust self at the end of the scan, so we don't have an incomplete set of # data if an exception occurs during the scan. self.servicesByName[serviceName] = service self.servicesById[serviceId] = service self.servicesByClass[service.service_class].append(service) return True def scan_dbus_service_getitems_done(self, serviceName, serviceId, values): # Keeping these exceptions for legacy reasons if serviceName == 'com.victronenergy.settings': di = 0 elif serviceName.startswith('com.victronenergy.vecan.'): di = 0 else: try: di = values['/DeviceInstance']['Value'] except KeyError: logger.info(" %s was skipped because it has no device instance" % serviceName) return False else: di = int(di) logger.info(" %s has device instance %s" % (serviceName, di)) service = self.make_service(serviceId, serviceName, di) paths = self.dbusTree.get('.'.join(serviceName.split('.')[0:3]), {}) for path, options in paths.items(): item = values.get(path, notfound) if item is notfound: service.paths[path] = self.make_monitor(service, path, None, None, options) else: service.set_seen(path) value = item.get('Value', None) text = item.get('Text', None) service.paths[path] = self.make_monitor(service, path, unwrap_dbus_value(value), unwrap_dbus_value(text), options) self.servicesByName[serviceName] = service self.servicesById[serviceId] = service self.servicesByClass[service.service_class].append(service) return True def handler_item_changes(self, items, senderId): if not isinstance(items, dict): return try: service = self.servicesById[senderId] except KeyError: # senderId isn't there, which means it hasn't been scanned yet. return for path, changes in items.items(): try: v = unwrap_dbus_value(changes['Value']) except (KeyError, TypeError): continue try: t = changes['Text'] except KeyError: t = str(v) self._handler_value_changes(service, path, v, t) def handler_value_changes(self, changes, path, senderId): # If this properyChange does not involve a value, our work is done. if 'Value' not in changes: return try: service = self.servicesById[senderId] except KeyError: # senderId isn't there, which means it hasn't been scanned yet. return v = unwrap_dbus_value(changes['Value']) # Some services don't send Text with their PropertiesChanged events. try: t = changes['Text'] except KeyError: t = str(v) self._handler_value_changes(service, path, v, t) def _handler_value_changes(self, service, path, value, text): try: a = service.paths[path] except KeyError: # path isn't there, which means it hasn't been scanned yet. return service.set_seen(path) # First update our store to the new value if a.value == value: return a.value = value a.text = text # And do the rest of the processing in on the mainloop if self.valueChangedCallback is not None: GLib.idle_add(exit_on_error, self._execute_value_changes, service.name, path, { 'Value': value, 'Text': text}, a.options) def _execute_value_changes(self, serviceName, objectPath, changes, options): # double check that the service still exists, as it might have # disappeared between scheduling-for and executing this function. if serviceName not in self.servicesByName: return self.valueChangedCallback(serviceName, objectPath, options, changes, self.get_device_instance(serviceName)) # Gets the value for a certain servicename and path # The default_value is returned when: # 1. When the service doesn't exist. # 2. When the path asked for isn't being monitored. # 3. When the path exists, but has dbus-invalid, ie an empty byte array. # 4. When the path asked for is being monitored, but doesn't exist for that service. def get_value(self, serviceName, objectPath, default_value=None): service = self.servicesByName.get(serviceName, None) if service is None: return default_value value = service.paths.get(objectPath, None) if value is None or value.value is None: return default_value return value.value # returns if a dbus exists now, by doing a blocking dbus call. # Typically seen will be sufficient and doesn't need access to the dbus. def exists(self, serviceName, objectPath): try: self.dbusConn.call_blocking(serviceName, objectPath, VE_INTERFACE, 'GetValue', '', []) return True except dbus.exceptions.DBusException as e: return False # Returns if there ever was a successful GetValue or valueChanged event. # Unlike get_value this return True also if the actual value is invalid. # # Note: the path might no longer exists anymore, but that doesn't happen in # practice. If a service really wants to reconfigure itself typically it should # reconnect to the dbus which causes it to be rescanned and seen will be updated. # If it is really needed to know if a path still exists, use exists. def seen(self, serviceName, objectPath): try: return self.servicesByName[serviceName].seen(objectPath) except KeyError: return False # Sets the value for a certain servicename and path, returns the return value of the D-Bus SetValue # method. If the underlying item does not exist (the service does not exist, or the objectPath was not # registered) the function will return -1 def set_value(self, serviceName, objectPath, value): # Check if the D-Bus object referenced by serviceName and objectPath is registered. There is no # necessity to do this, but it is in line with previous implementations which kept VeDbusItemImport # objects for registers items only. service = self.servicesByName.get(serviceName, None) if service is None: return -1 if objectPath not in service.paths: return -1 # We do not catch D-Bus exceptions here, because the previous implementation did not do that either. return self.dbusConn.call_blocking(serviceName, objectPath, dbus_interface=VE_INTERFACE, method='SetValue', signature=None, args=[wrap_dbus_value(value)]) # Similar to set_value, but operates asynchronously def set_value_async(self, serviceName, objectPath, value, reply_handler=None, error_handler=None): service = self.servicesByName.get(serviceName, None) if service is not None: if objectPath in service.paths: self.dbusConn.call_async(serviceName, objectPath, dbus_interface=VE_INTERFACE, method='SetValue', signature=None, args=[wrap_dbus_value(value)], reply_handler=reply_handler, error_handler=error_handler) return if error_handler is not None: error_handler(TypeError('Service or path not found, ' 'service=%s, path=%s' % (serviceName, objectPath))) # returns a dictionary, keys are the servicenames, value the instances # optionally use the classfilter to get only a certain type of services, for # example com.victronenergy.battery. def get_service_list(self, classfilter=None): if classfilter is None: return { servicename: service.deviceInstance \ for servicename, service in self.servicesByName.items() } if classfilter not in self.servicesByClass: return {} return { service.name: service.deviceInstance \ for service in self.servicesByClass[classfilter] } def get_device_instance(self, serviceName): return self.servicesByName[serviceName].deviceInstance def track_value(self, serviceName, objectPath, callback, *args, **kwargs): """ A DbusMonitor can watch specific service/path combos for changes so that it is not fully reliant on the global handler_value_changes in this class. Additional watches are deleted automatically when the service disappears from dbus. """ cb = partial(callback, *args, **kwargs) def root_tracker(items): # Check if objectPath in dict try: v = items[objectPath] _v = unwrap_dbus_value(v['Value']) except (KeyError, TypeError): return # not in this dict try: t = v['Text'] except KeyError: cb({'Value': _v }) else: cb({'Value': _v, 'Text': t}) # Track changes on the path, and also on root self.serviceWatches[serviceName].extend(( self.dbusConn.add_signal_receiver(cb, dbus_interface='com.victronenergy.BusItem', signal_name='PropertiesChanged', path=objectPath, bus_name=serviceName), self.dbusConn.add_signal_receiver(root_tracker, dbus_interface='com.victronenergy.BusItem', signal_name='ItemsChanged', path="/", bus_name=serviceName), )) # ====== ALL CODE BELOW THIS LINE IS PURELY FOR DEVELOPING THIS CLASS ====== # Example function that can be used as a starting point to use this code def value_changed_on_dbus(dbusServiceName, dbusPath, dict, changes, deviceInstance): logger.debug("0 ----------------") logger.debug("1 %s%s changed" % (dbusServiceName, dbusPath)) logger.debug("2 vrm dict : %s" % dict) logger.debug("3 changes-text: %s" % changes['Text']) logger.debug("4 changes-value: %s" % changes['Value']) logger.debug("5 deviceInstance: %s" % deviceInstance) logger.debug("6 - end") def nameownerchange(a, b): # used to find memory leaks in dbusmonitor and VeDbusItemImport import gc gc.collect() objects = gc.get_objects() print (len([o for o in objects if type(o).__name__ == 'VeDbusItemImport'])) print (len([o for o in objects if type(o).__name__ == 'SignalMatch'])) print (len(objects)) def print_values(dbusmonitor): a = dbusmonitor.get_value('wrongservice', '/DbusInvalid', default_value=1000) b = dbusmonitor.get_value('com.victronenergy.dummyservice.ttyO1', '/NotInTheMonitorList', default_value=1000) c = dbusmonitor.get_value('com.victronenergy.dummyservice.ttyO1', '/DbusInvalid', default_value=1000) d = dbusmonitor.get_value('com.victronenergy.dummyservice.ttyO1', '/NonExistingButMonitored', default_value=1000) print ("All should be 1000: Wrong Service: %s, NotInTheMonitorList: %s, DbusInvalid: %s, NonExistingButMonitored: %s" % (a, b, c, d)) return True # We have a mainloop, but that is just for developing this code. Normally above class & code is used from # some other class, such as vrmLogger or the pubsub Implementation. def main(): # Init logging logging.basicConfig(level=logging.DEBUG) logger.info(__file__ + " is starting up") # Have a mainloop, so we can send/receive asynchronous calls to and from dbus DBusGMainLoop(set_as_default=True) import os import sys sys.path.insert(1, os.path.join(os.path.dirname(__file__), '../../')) dummy = {'code': None, 'whenToLog': 'configChange', 'accessLevel': None} monitorlist = {'com.victronenergy.dummyservice': { '/Connected': dummy, '/ProductName': dummy, '/Mgmt/Connection': dummy, '/Dc/0/Voltage': dummy, '/Dc/0/Current': dummy, '/Dc/0/Temperature': dummy, '/Load/I': dummy, '/FirmwareVersion': dummy, '/DbusInvalid': dummy, '/NonExistingButMonitored': dummy}} d = DbusMonitor(monitorlist, value_changed_on_dbus, deviceAddedCallback=nameownerchange, deviceRemovedCallback=nameownerchange) GLib.timeout_add(1000, print_values, d) # Start and run the mainloop logger.info("Starting mainloop, responding on only events") mainloop = GLib.MainLoop() mainloop.run() if __name__ == "__main__": main() ================================================ FILE: velib_python/velib_python/latest/oldestVersion ================================================ v3.50 ================================================ FILE: velib_python/velib_python/latest/settingsdevice.py ================================================ import dbus import logging import time from functools import partial # Local imports from vedbus import VeDbusItemImport ## Indexes for the setting dictonary. PATH = 0 VALUE = 1 MINIMUM = 2 MAXIMUM = 3 SILENT = 4 ## The Settings Device class. # Used by python programs, such as the vrm-logger, to read and write settings they # need to store on disk. And since these settings might be changed from a different # source, such as the GUI, the program can pass an eventCallback that will be called # as soon as some setting is changed. # # The settings are stored in flash via the com.victronenergy.settings service on dbus. # See https://github.com/victronenergy/localsettings for more info. # # If there are settings in de supportSettings list which are not yet on the dbus, # and therefore not yet in the xml file, they will be added through the dbus-addSetting # interface of com.victronenergy.settings. class SettingsDevice(object): ## The constructor processes the tree of dbus-items. # @param bus the system-dbus object # @param name the dbus-service-name of the settings dbus service, 'com.victronenergy.settings' # @param supportedSettings dictionary with all setting-names, and their defaultvalue, min, max and whether # the setting is silent. The 'silent' entry is optional. If set to true, no changes in the setting will # be logged by localsettings. # @param eventCallback function that will be called on changes on any of these settings # @param timeout Maximum interval to wait for localsettings. An exception is thrown at the end of the # interval if the localsettings D-Bus service has not appeared yet. def __init__(self, bus, supportedSettings, eventCallback, name='com.victronenergy.settings', timeout=0): logging.debug("===== Settings device init starting... =====") self._bus = bus self._dbus_name = name self._eventCallback = eventCallback self._values = {} # stored the values, used to pass the old value along on a setting change self._settings = {} count = 0 while True: if 'com.victronenergy.settings' in self._bus.list_names(): break if count == timeout: raise Exception("The settings service com.victronenergy.settings does not exist!") count += 1 logging.info('waiting for settings') time.sleep(1) # Add the items. self.addSettings(supportedSettings) logging.debug("===== Settings device init finished =====") def addSettings(self, settings): for setting, options in settings.items(): silent = len(options) > SILENT and options[SILENT] busitem = self.addSetting(options[PATH], options[VALUE], options[MINIMUM], options[MAXIMUM], silent, callback=partial(self.handleChangedSetting, setting)) self._settings[setting] = busitem self._values[setting] = busitem.get_value() def addSetting(self, path, value, _min, _max, silent=False, callback=None): busitem = VeDbusItemImport(self._bus, self._dbus_name, path, callback) if busitem.exists and (value, _min, _max, silent) == busitem._proxy.GetAttributes(): logging.debug("Setting %s found" % path) else: logging.info("Setting %s does not exist yet or must be adjusted" % path) # Prepare to add the setting. Most dbus types extend the python # type so it is only necessary to additionally test for Int64. if isinstance(value, (int, dbus.Int64)): itemType = 'i' elif isinstance(value, float): itemType = 'f' else: itemType = 's' # Add the setting # TODO, make an object that inherits VeDbusItemImport, and complete the D-Bus settingsitem interface settings_item = VeDbusItemImport(self._bus, self._dbus_name, '/Settings', createsignal=False) setting_path = path.replace('/Settings/', '', 1) if silent: settings_item._proxy.AddSilentSetting('', setting_path, value, itemType, _min, _max) else: settings_item._proxy.AddSetting('', setting_path, value, itemType, _min, _max) busitem = VeDbusItemImport(self._bus, self._dbus_name, path, callback) return busitem def handleChangedSetting(self, setting, servicename, path, changes): oldvalue = self._values[setting] if setting in self._values else None self._values[setting] = changes['Value'] if self._eventCallback is None: return self._eventCallback(setting, oldvalue, changes['Value']) def setDefault(self, path): item = VeDbusItemImport(self._bus, self._dbus_name, path, createsignal=False) item.set_default() def __getitem__(self, setting): return self._settings[setting].get_value() def __setitem__(self, setting, newvalue): result = self._settings[setting].set_value(newvalue) if result != 0: # Trying to make some false change to our own settings? How dumb! assert False ================================================ FILE: velib_python/velib_python/latest/ve_utils.py ================================================ #!/usr/bin/env python3 # -*- coding: utf-8 -*- import sys from traceback import print_exc from os import _exit as os_exit from os import statvfs from subprocess import check_output, CalledProcessError import logging import dbus logger = logging.getLogger(__name__) VEDBUS_INVALID = dbus.Array([], signature=dbus.Signature('i'), variant_level=1) class NoVrmPortalIdError(Exception): pass # Use this function to make sure the code quits on an unexpected exception. Make sure to use it # when using GLib.idle_add and also GLib.timeout_add. # Without this, the code will just keep running, since GLib does not stop the mainloop on an # exception. # Example: GLib.idle_add(exit_on_error, myfunc, arg1, arg2) def exit_on_error(func, *args, **kwargs): try: return func(*args, **kwargs) except: try: print ('exit_on_error: there was an exception. Printing stacktrace will be tried and then exit') print_exc() except: pass # sys.exit() is not used, since that throws an exception, which does not lead to a program # halt when used in a dbus callback, see connection.py in the Python/Dbus libraries, line 230. os_exit(1) __vrm_portal_id = None def get_vrm_portal_id(): # The original definition of the VRM Portal ID is that it is the mac # address of the onboard- ethernet port (eth0), stripped from its colons # (:) and lower case. This may however differ between platforms. On Venus # the task is therefore deferred to /sbin/get-unique-id so that a # platform specific method can be easily defined. # # If /sbin/get-unique-id does not exist, then use the ethernet address # of eth0. This also handles the case where velib_python is used as a # package install on a Raspberry Pi. # # On a Linux host where the network interface may not be eth0, you can set # the VRM_IFACE environment variable to the correct name. global __vrm_portal_id if __vrm_portal_id: return __vrm_portal_id portal_id = None # First try the method that works if we don't have a data partition. This # will fail when the current user is not root. try: portal_id = check_output("/sbin/get-unique-id").decode("utf-8", "ignore").strip() if not portal_id: raise NoVrmPortalIdError("get-unique-id returned blank") __vrm_portal_id = portal_id return portal_id except CalledProcessError: # get-unique-id returned non-zero raise NoVrmPortalIdError("get-unique-id returned non-zero") except OSError: # File doesn't exist, use fallback pass # Fall back to getting our id using a syscall. Assume we are on linux. # Allow the user to override what interface is used using an environment # variable. import fcntl, socket, struct, os iface = os.environ.get('VRM_IFACE', 'eth0').encode('ascii') s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) try: info = fcntl.ioctl(s.fileno(), 0x8927, struct.pack('256s', iface[:15])) except IOError: raise NoVrmPortalIdError("ioctl failed for eth0") __vrm_portal_id = info[18:24].hex() return __vrm_portal_id # See VE.Can registers - public.docx for definition of this conversion def convert_vreg_version_to_readable(version): def str_to_arr(x, length): a = [] for i in range(0, len(x), length): a.append(x[i:i+length]) return a x = "%x" % version x = x.upper() if len(x) == 5 or len(x) == 3 or len(x) == 1: x = '0' + x a = str_to_arr(x, 2); # remove the first 00 if there are three bytes and it is 00 if len(a) == 3 and a[0] == '00': a.remove(0); # if we have two or three bytes now, and the first character is a 0, remove it if len(a) >= 2 and a[0][0:1] == '0': a[0] = a[0][1]; result = '' for item in a: result += ('.' if result != '' else '') + item result = 'v' + result return result def get_free_space(path): result = -1 try: s = statvfs(path) result = s.f_frsize * s.f_bavail # Number of free bytes that ordinary users except Exception as ex: logger.info("Error while retrieving free space for path %s: %s" % (path, ex)) return result def _get_sysfs_machine_name(): try: with open('/sys/firmware/devicetree/base/model', 'r') as f: return f.read().rstrip('\x00') except IOError: pass return None # Returns None if it cannot find a machine name. Otherwise returns the string # containing the name def get_machine_name(): # First try calling the venus utility script try: return check_output("/usr/bin/product-name").strip().decode('UTF-8') except (CalledProcessError, OSError): pass # Fall back to sysfs name = _get_sysfs_machine_name() if name is not None: return name # Fall back to venus build machine name try: with open('/etc/venus/machine', 'r', encoding='UTF-8') as f: return f.read().strip() except IOError: pass return None def get_product_id(): """ Find the machine ID and return it. """ # First try calling the venus utility script try: return check_output("/usr/bin/product-id").strip().decode('UTF-8') except (CalledProcessError, OSError): pass # Fall back machine name mechanism name = _get_sysfs_machine_name() return { 'Color Control GX': 'C001', 'Venus GX': 'C002', 'Octo GX': 'C006', 'EasySolar-II': 'C007', 'MultiPlus-II': 'C008', 'Maxi GX': 'C009', 'Cerbo GX': 'C00A' }.get(name, 'C003') # C003 is Generic # Returns False if it cannot open the file. Otherwise returns its rstripped contents def read_file(path): content = False try: with open(path, 'r') as f: content = f.read().rstrip() except Exception as ex: logger.debug("Error while reading %s: %s" % (path, ex)) return content def wrap_dbus_value(value): if value is None: return VEDBUS_INVALID if isinstance(value, float): return dbus.Double(value, variant_level=1) if isinstance(value, bool): return dbus.Boolean(value, variant_level=1) if isinstance(value, int): try: return dbus.Int32(value, variant_level=1) except OverflowError: return dbus.Int64(value, variant_level=1) if isinstance(value, str): return dbus.String(value, variant_level=1) if isinstance(value, list): if len(value) == 0: # If the list is empty we cannot infer the type of the contents. So assume unsigned integer. # A (signed) integer is dangerous, because an empty list of signed integers is used to encode # an invalid value. return dbus.Array([], signature=dbus.Signature('u'), variant_level=1) return dbus.Array([wrap_dbus_value(x) for x in value], variant_level=1) if isinstance(value, dict): # Wrapping the keys of the dictionary causes D-Bus errors like: # 'arguments to dbus_message_iter_open_container() were incorrect, # assertion "(type == DBUS_TYPE_ARRAY && contained_signature && # *contained_signature == DBUS_DICT_ENTRY_BEGIN_CHAR) || (contained_signature == NULL || # _dbus_check_is_valid_signature (contained_signature))" failed in file ...' return dbus.Dictionary({(k, wrap_dbus_value(v)) for k, v in value.items()}, variant_level=1) return value dbus_int_types = (dbus.Int32, dbus.UInt32, dbus.Byte, dbus.Int16, dbus.UInt16, dbus.UInt32, dbus.Int64, dbus.UInt64) def unwrap_dbus_value(val): """Converts D-Bus values back to the original type. For example if val is of type DBus.Double, a float will be returned.""" if isinstance(val, dbus_int_types): return int(val) if isinstance(val, dbus.Double): return float(val) if isinstance(val, dbus.Array): v = [unwrap_dbus_value(x) for x in val] return None if len(v) == 0 else v if isinstance(val, (dbus.Signature, dbus.String)): return str(val) # Python has no byte type, so we convert to an integer. if isinstance(val, dbus.Byte): return int(val) if isinstance(val, dbus.ByteArray): return "".join([bytes(x) for x in val]) if isinstance(val, (list, tuple)): return [unwrap_dbus_value(x) for x in val] if isinstance(val, (dbus.Dictionary, dict)): # Do not unwrap the keys, see comment in wrap_dbus_value return dict([(x, unwrap_dbus_value(y)) for x, y in val.items()]) if isinstance(val, dbus.Boolean): return bool(val) return val # When supported, only name owner changes for the the given namespace are reported. This # prevents spending cpu time at irrelevant changes, like scripts accessing the bus temporarily. def add_name_owner_changed_receiver(dbus, name_owner_changed, namespace="com.victronenergy"): # support for arg0namespace is submitted upstream, but not included at the time of # writing, Venus OS does support it, so try if it works. if namespace is None: dbus.add_signal_receiver(name_owner_changed, signal_name='NameOwnerChanged') else: try: dbus.add_signal_receiver(name_owner_changed, signal_name='NameOwnerChanged', arg0namespace=namespace) except TypeError: dbus.add_signal_receiver(name_owner_changed, signal_name='NameOwnerChanged') ================================================ FILE: velib_python/velib_python/latest/vedbus.py ================================================ #!/usr/bin/env python3 # -*- coding: utf-8 -*- import dbus.service import logging import traceback import os import weakref from collections import defaultdict from ve_utils import wrap_dbus_value, unwrap_dbus_value # vedbus contains three classes: # VeDbusItemImport -> use this to read data from the dbus, ie import # VeDbusItemExport -> use this to export data to the dbus (one value) # VeDbusService -> use that to create a service and export several values to the dbus # Code for VeDbusItemImport is copied from busitem.py and thereafter modified. # All projects that used busitem.py need to migrate to this package. And some # projects used to define there own equivalent of VeDbusItemExport. Better to # use VeDbusItemExport, or even better the VeDbusService class that does it all for you. # TODOS # 1 check for datatypes, it works now, but not sure if all is compliant with # com.victronenergy.BusItem interface definition. See also the files in # tests_and_examples. And see 'if type(v) == dbus.Byte:' on line 102. Perhaps # something similar should also be done in VeDbusBusItemExport? # 2 Shouldn't VeDbusBusItemExport inherit dbus.service.Object? # 7 Make hard rules for services exporting data to the D-Bus, in order to make tracking # changes possible. Does everybody first invalidate its data before leaving the bus? # And what about before taking one object away from the bus, instead of taking the # whole service offline? # They should! And after taking one value away, do we need to know that someone left # the bus? Or we just keep that value in invalidated for ever? Result is that we can't # see the difference anymore between an invalidated value and a value that was first on # the bus and later not anymore. See comments above VeDbusItemImport as well. # 9 there are probably more todos in the code below. # Some thoughts with regards to the data types: # # Text from: http://dbus.freedesktop.org/doc/dbus-python/doc/tutorial.html#data-types # --- # Variants are represented by setting the variant_level keyword argument in the # constructor of any D-Bus data type to a value greater than 0 (variant_level 1 # means a variant containing some other data type, variant_level 2 means a variant # containing a variant containing some other data type, and so on). If a non-variant # is passed as an argument but introspection indicates that a variant is expected, # it'll automatically be wrapped in a variant. # --- # # Also the different dbus datatypes, such as dbus.Int32, and dbus.UInt32 are a subclass # of Python int. dbus.String is a subclass of Python standard class unicode, etcetera # # So all together that explains why we don't need to explicitly convert back and forth # between the dbus datatypes and the standard python datatypes. Note that all datatypes # in python are objects. Even an int is an object. # The signature of a variant is 'v'. # Export ourselves as a D-Bus service. class VeDbusService(object): def __init__(self, servicename, bus=None, register=True): # dict containing the VeDbusItemExport objects, with their path as the key. self._dbusobjects = {} self._dbusnodes = {} self._ratelimiters = [] self._dbusname = None self.name = servicename # dict containing the onchange callbacks, for each object. Object path is the key self._onchangecallbacks = {} # Connect to session bus whenever present, else use the system bus self._dbusconn = bus or (dbus.SessionBus() if 'DBUS_SESSION_BUS_ADDRESS' in os.environ else dbus.SystemBus()) # make the dbus connection available to outside, could make this a true property instead, but ach.. self.dbusconn = self._dbusconn # Add the root item that will return all items as a tree self._dbusnodes['/'] = VeDbusRootExport(self._dbusconn, '/', self) # Immediately register the service unless requested not to if register: logging.warning("USING OUTDATED REGISTRATION METHOD!") logging.warning("Please set register=False, then call the register method " "after adding all mandatory paths. See " "https://github.com/victronenergy/venus/wiki/dbus-api") self.register() def register(self): # Register ourselves on the dbus, trigger an error if already in use (do_not_queue) self._dbusname = dbus.service.BusName(self.name, self._dbusconn, do_not_queue=True) logging.info("registered ourselves on D-Bus as %s" % self.name) # To force immediate deregistering of this dbus service and all its object paths, explicitly # call __del__(). def __del__(self): for node in list(self._dbusnodes.values()): node.__del__() self._dbusnodes.clear() for item in list(self._dbusobjects.values()): item.__del__() self._dbusobjects.clear() if self._dbusname: self._dbusname.__del__() # Forces call to self._bus.release_name(self._name), see source code self._dbusname = None def get_name(self): return self._dbusname.get_name() # @param callbackonchange function that will be called when this value is changed. First parameter will # be the path of the object, second the new value. This callback should return # True to accept the change, False to reject it. def add_path(self, path, value, description="", writeable=False, onchangecallback=None, gettextcallback=None, valuetype=None, itemtype=None): if onchangecallback is not None: self._onchangecallbacks[path] = onchangecallback itemtype = itemtype or VeDbusItemExport item = itemtype(self._dbusconn, path, value, description, writeable, self._value_changed, gettextcallback, deletecallback=self._item_deleted, valuetype=valuetype) spl = path.split('/') for i in range(2, len(spl)): subPath = '/'.join(spl[:i]) if subPath not in self._dbusnodes and subPath not in self._dbusobjects: self._dbusnodes[subPath] = VeDbusTreeExport(self._dbusconn, subPath, self) self._dbusobjects[path] = item logging.debug('added %s with start value %s. Writeable is %s' % (path, value, writeable)) return item # Add the mandatory paths, as per victron dbus api doc def add_mandatory_paths(self, processname, processversion, connection, deviceinstance, productid, productname, firmwareversion, hardwareversion, connected): self.add_path('/Mgmt/ProcessName', processname) self.add_path('/Mgmt/ProcessVersion', processversion) self.add_path('/Mgmt/Connection', connection) # Create rest of the mandatory objects self.add_path('/DeviceInstance', deviceinstance) self.add_path('/ProductId', productid) self.add_path('/ProductName', productname) self.add_path('/FirmwareVersion', firmwareversion) self.add_path('/HardwareVersion', hardwareversion) self.add_path('/Connected', connected) # Callback function that is called from the VeDbusItemExport objects when a value changes. This function # maps the change-request to the onchangecallback given to us for this specific path. def _value_changed(self, path, newvalue): if path not in self._onchangecallbacks: return True return self._onchangecallbacks[path](path, newvalue) def _item_deleted(self, path): self._dbusobjects.pop(path) for np in list(self._dbusnodes.keys()): if np != '/': for ip in self._dbusobjects: if ip.startswith(np + '/'): break else: self._dbusnodes[np].__del__() self._dbusnodes.pop(np) def __getitem__(self, path): return self._dbusobjects[path].local_get_value() def __setitem__(self, path, newvalue): self._dbusobjects[path].local_set_value(newvalue) def __delitem__(self, path): self._dbusobjects[path].__del__() # Invalidates and then removes the object path assert path not in self._dbusobjects def __contains__(self, path): return path in self._dbusobjects def __enter__(self): l = ServiceContext(self) self._ratelimiters.append(l) return l def __exit__(self, *exc): # pop off the top one and flush it. If with statements are nested # then each exit flushes its own part. if self._ratelimiters: self._ratelimiters.pop().flush() class ServiceContext(object): def __init__(self, parent): self.parent = parent self.changes = {} def __contains__(self, path): return path in self.parent def __getitem__(self, path): return self.parent[path] def __setitem__(self, path, newvalue): c = self.parent._dbusobjects[path]._local_set_value(newvalue) if c is not None: self.changes[path] = c def __delitem__(self, path): if path in self.changes: del self.changes[path] del self.parent[path] def flush(self): if self.changes: self.parent._dbusnodes['/'].ItemsChanged(self.changes) self.changes.clear() def add_path(self, path, value, *args, **kwargs): self.parent.add_path(path, value, *args, **kwargs) self.changes[path] = { 'Value': wrap_dbus_value(value), 'Text': self.parent._dbusobjects[path].GetText() } def del_tree(self, root): root = root.rstrip('/') for p in list(self.parent._dbusobjects.keys()): if p == root or p.startswith(root + '/'): self[p] = None self.parent._dbusobjects[p].__del__() def get_name(self): return self.parent.get_name() class TrackerDict(defaultdict): """ Same as defaultdict, but passes the key to default_factory. """ def __missing__(self, key): self[key] = x = self.default_factory(key) return x class VeDbusRootTracker(object): """ This tracks the root of a dbus path and listens for PropertiesChanged signals. When a signal arrives, parse it and unpack the key/value changes into traditional events, then pass it to the original eventCallback method. """ def __init__(self, bus, serviceName): self.importers = defaultdict(weakref.WeakSet) self.serviceName = serviceName self._match = bus.get_object(serviceName, '/', introspect=False).connect_to_signal( "ItemsChanged", weak_functor(self._items_changed_handler)) def __del__(self): self._match.remove() self._match = None def add(self, i): self.importers[i.path].add(i) def _items_changed_handler(self, items): if not isinstance(items, dict): return for path, changes in items.items(): try: v = changes['Value'] except KeyError: continue try: t = changes['Text'] except KeyError: t = str(unwrap_dbus_value(v)) for i in self.importers.get(path, ()): i._properties_changed_handler({'Value': v, 'Text': t}) """ Importing basics: - If when we power up, the D-Bus service does not exist, or it does exist and the path does not yet exist, still subscribe to a signal: as soon as it comes online it will send a signal with its initial value, which VeDbusItemImport will receive and use to update local cache. And, when set, call the eventCallback. - If when we power up, save it - When using get_value, know that there is no difference between services (or object paths) that don't exist and paths that are invalid (= empty array, see above). Both will return None. In case you do really want to know ifa path exists or not, use the exists property. - When a D-Bus service leaves the D-Bus, it will first invalidate all its values, and send signals with that update, and only then leave the D-Bus. (or do we need to subscribe to the NameOwnerChanged- signal!?!) To be discussed and make sure. Not really urgent, since all existing code that uses this class already subscribes to the NameOwnerChanged signal, and subsequently removes instances of this class. Read when using this class: Note that when a service leaves that D-Bus without invalidating all its exported objects first, for example because it is killed, VeDbusItemImport doesn't have a clue. So when using VeDbusItemImport, make sure to also subscribe to the NamerOwnerChanged signal on bus-level. Or just use dbusmonitor, because that takes care of all of that for you. """ class VeDbusItemImport(object): def __new__(cls, bus, serviceName, path, eventCallback=None, createsignal=True): instance = object.__new__(cls) # If signal tracking should be done, also add to root tracker if createsignal: if "_roots" not in cls.__dict__: cls._roots = TrackerDict(lambda k: VeDbusRootTracker(bus, k)) return instance ## Constructor # @param bus the bus-object (SESSION or SYSTEM). # @param serviceName the dbus-service-name (string), for example 'com.victronenergy.battery.ttyO1' # @param path the object-path, for example '/Dc/V' # @param eventCallback function that you want to be called on a value change # @param createSignal only set this to False if you use this function to one time read a value. When # leaving it to True, make sure to also subscribe to the NameOwnerChanged signal # elsewhere. See also note some 15 lines up. def __init__(self, bus, serviceName, path, eventCallback=None, createsignal=True): # TODO: is it necessary to store _serviceName and _path? Isn't it # stored in the bus_getobjectsomewhere? self._serviceName = serviceName self._path = path self._match = None # TODO: _proxy is being used in settingsdevice.py, make a getter for that self._proxy = bus.get_object(serviceName, path, introspect=False) self.eventCallback = eventCallback assert eventCallback is None or createsignal == True if createsignal: self._match = self._proxy.connect_to_signal( "PropertiesChanged", weak_functor(self._properties_changed_handler)) self._roots[serviceName].add(self) # store the current value in _cachedvalue. When it doesn't exists set _cachedvalue to # None, same as when a value is invalid self._cachedvalue = None try: v = self._proxy.GetValue() except dbus.exceptions.DBusException: pass else: self._cachedvalue = unwrap_dbus_value(v) def __del__(self): if self._match is not None: self._match.remove() self._match = None self._proxy = None def _refreshcachedvalue(self): self._cachedvalue = unwrap_dbus_value(self._proxy.GetValue()) ## Returns the path as a string, for example '/AC/L1/V' @property def path(self): return self._path ## Returns the dbus service name as a string, for example com.victronenergy.vebus.ttyO1 @property def serviceName(self): return self._serviceName ## Returns the value of the dbus-item. # the type will be a dbus variant, for example dbus.Int32(0, variant_level=1) # this is not a property to keep the name consistant with the com.victronenergy.busitem interface # returns None when the property is invalid def get_value(self): return self._cachedvalue ## Writes a new value to the dbus-item def set_value(self, newvalue): r = self._proxy.SetValue(wrap_dbus_value(newvalue)) # instead of just saving the value, go to the dbus and get it. So we have the right type etc. if r == 0: self._refreshcachedvalue() return r ## Resets the item to its default value def set_default(self): self._proxy.SetDefault() self._refreshcachedvalue() ## Returns the text representation of the value. # For example when the value is an enum/int GetText might return the string # belonging to that enum value. Another example, for a voltage, GetValue # would return a float, 12.0Volt, and GetText could return 12 VDC. # # Note that this depends on how the dbus-producer has implemented this. def get_text(self): return self._proxy.GetText() ## Returns true of object path exists, and false if it doesn't @property def exists(self): # TODO: do some real check instead of this crazy thing. r = False try: r = self._proxy.GetValue() r = True except dbus.exceptions.DBusException: pass return r ## callback for the trigger-event. # @param eventCallback the event-callback-function. @property def eventCallback(self): return self._eventCallback @eventCallback.setter def eventCallback(self, eventCallback): self._eventCallback = eventCallback ## Is called when the value of the imported bus-item changes. # Stores the new value in our local cache, and calls the eventCallback, if set. def _properties_changed_handler(self, changes): if "Value" in changes: changes['Value'] = unwrap_dbus_value(changes['Value']) self._cachedvalue = changes['Value'] if self._eventCallback: # The reason behind this try/except is to prevent errors silently ending up the an error # handler in the dbus code. try: self._eventCallback(self._serviceName, self._path, changes) except: traceback.print_exc() os._exit(1) # sys.exit() is not used, since that also throws an exception class VeDbusTreeExport(dbus.service.Object): def __init__(self, bus, objectPath, service): dbus.service.Object.__init__(self, bus, objectPath) self._service = service logging.debug("VeDbusTreeExport %s has been created" % objectPath) def __del__(self): # self._get_path() will raise an exception when retrieved after the call to .remove_from_connection, # so we need a copy. path = self._get_path() if path is None: return self.remove_from_connection() logging.debug("VeDbusTreeExport %s has been removed" % path) def _get_path(self): if len(self._locations) == 0: return None return self._locations[0][1] def _get_value_handler(self, path, get_text=False): logging.debug("_get_value_handler called for %s" % path) r = {} px = path if not px.endswith('/'): px += '/' for p, item in self._service._dbusobjects.items(): if p.startswith(px): v = item.GetText() if get_text else wrap_dbus_value(item.local_get_value()) r[p[len(px):]] = v logging.debug(r) return r @dbus.service.method('com.victronenergy.BusItem', out_signature='v') def GetValue(self): value = self._get_value_handler(self._get_path()) return dbus.Dictionary(value, signature=dbus.Signature('sv'), variant_level=1) @dbus.service.method('com.victronenergy.BusItem', out_signature='v') def GetText(self): return self._get_value_handler(self._get_path(), True) def local_get_value(self): return self._get_value_handler(self.path) class VeDbusRootExport(VeDbusTreeExport): @dbus.service.signal('com.victronenergy.BusItem', signature='a{sa{sv}}') def ItemsChanged(self, changes): pass @dbus.service.method('com.victronenergy.BusItem', out_signature='a{sa{sv}}') def GetItems(self): return { path: { 'Value': wrap_dbus_value(item.local_get_value()), 'Text': item.GetText() } for path, item in self._service._dbusobjects.items() } class VeDbusItemExport(dbus.service.Object): ## Constructor of VeDbusItemExport # # Use this object to export (publish), values on the dbus # Creates the dbus-object under the given dbus-service-name. # @param bus The dbus object. # @param objectPath The dbus-object-path. # @param value Value to initialize ourselves with, defaults to None which means Invalid # @param description String containing a description. Can be called over the dbus with GetDescription() # @param writeable what would this do!? :). # @param callback Function that will be called when someone else changes the value of this VeBusItem # over the dbus. First parameter passed to callback will be our path, second the new # value. This callback should return True to accept the change, False to reject it. def __init__(self, bus, objectPath, value=None, description=None, writeable=False, onchangecallback=None, gettextcallback=None, deletecallback=None, valuetype=None): dbus.service.Object.__init__(self, bus, objectPath) self._onchangecallback = onchangecallback self._gettextcallback = gettextcallback self._value = value self._description = description self._writeable = writeable self._deletecallback = deletecallback self._type = valuetype # To force immediate deregistering of this dbus object, explicitly call __del__(). def __del__(self): # self._get_path() will raise an exception when retrieved after the # call to .remove_from_connection, so we need a copy. path = self._get_path() if path == None: return if self._deletecallback is not None: self._deletecallback(path) self.remove_from_connection() logging.debug("VeDbusItemExport %s has been removed" % path) def _get_path(self): if len(self._locations) == 0: return None return self._locations[0][1] ## Sets the value. And in case the value is different from what it was, a signal # will be emitted to the dbus. This function is to be used in the python code that # is using this class to export values to the dbus. # set value to None to indicate that it is Invalid def local_set_value(self, newvalue): changes = self._local_set_value(newvalue) if changes is not None: self.PropertiesChanged(changes) def _local_set_value(self, newvalue): if self._value == newvalue: return None self._value = newvalue return { 'Value': wrap_dbus_value(newvalue), 'Text': self.GetText() } def local_get_value(self): return self._value # ==== ALL FUNCTIONS BELOW THIS LINE WILL BE CALLED BY OTHER PROCESSES OVER THE DBUS ==== ## Dbus exported method SetValue # Function is called over the D-Bus by other process. It will first check (via callback) if new # value is accepted. And it is, stores it and emits a changed-signal. # @param value The new value. # @return completion-code When successful a 0 is return, and when not a -1 is returned. @dbus.service.method('com.victronenergy.BusItem', in_signature='v', out_signature='i') def SetValue(self, newvalue): if not self._writeable: return 1 # NOT OK newvalue = unwrap_dbus_value(newvalue) # If value type is enforced, cast it. If the type can be coerced # python will do it for us. This allows ints to become floats, # or bools to become ints. Additionally also allow None, so that # a path may be invalidated. if self._type is not None and newvalue is not None: try: newvalue = self._type(newvalue) except (ValueError, TypeError): return 1 # NOT OK if newvalue == self._value: return 0 # OK # call the callback given to us, and check if new value is OK. if (self._onchangecallback is None or (self._onchangecallback is not None and self._onchangecallback(self.__dbus_object_path__, newvalue))): self.local_set_value(newvalue) return 0 # OK return 2 # NOT OK ## Dbus exported method GetDescription # # Returns the a description. # @param language A language code (e.g. ISO 639-1 en-US). # @param length Lenght of the language string. # @return description @dbus.service.method('com.victronenergy.BusItem', in_signature='si', out_signature='s') def GetDescription(self, language, length): return self._description if self._description is not None else 'No description given' ## Dbus exported method GetValue # Returns the value. # @return the value when valid, and otherwise an empty array @dbus.service.method('com.victronenergy.BusItem', out_signature='v') def GetValue(self): return wrap_dbus_value(self._value) ## Dbus exported method GetText # Returns the value as string of the dbus-object-path. # @return text A text-value. '---' when local value is invalid @dbus.service.method('com.victronenergy.BusItem', out_signature='s') def GetText(self): if self._value is None: return '---' # Default conversion from dbus.Byte will get you a character (so 'T' instead of '84'), so we # have to convert to int first. Note that if a dbus.Byte turns up here, it must have come from # the application itself, as all data from the D-Bus should have been unwrapped by now. if self._gettextcallback is None and type(self._value) == dbus.Byte: return str(int(self._value)) if self._gettextcallback is None and self.__dbus_object_path__ == '/ProductId': return "0x%X" % self._value if self._gettextcallback is None: return str(self._value) return self._gettextcallback(self.__dbus_object_path__, self._value) ## The signal that indicates that the value has changed. # Other processes connected to this BusItem object will have subscribed to the # event when they want to track our state. @dbus.service.signal('com.victronenergy.BusItem', signature='a{sv}') def PropertiesChanged(self, changes): pass ## This class behaves like a regular reference to a class method (eg. self.foo), but keeps a weak reference ## to the object which method is to be called. ## Use this object to break circular references. class weak_functor: def __init__(self, f): self._r = weakref.ref(f.__self__) self._f = weakref.ref(f.__func__) def __call__(self, *args, **kargs): r = self._r() f = self._f() if r == None or f == None: return f(r, *args, **kargs) ================================================ FILE: velib_python/velib_python/v3.34/dbusmonitor.py ================================================ #!/usr/bin/env python3 # -*- coding: utf-8 -*- ## @package dbus_vrm # This code takes care of the D-Bus interface (not all of below is implemented yet): # - on startup it scans the dbus for services we know. For each known service found, it searches for # objects/paths we know. Everything we find is stored in items{}, and an event is registered: if a # value changes weĺl be notified and can pass that on to our owner. For example the vrmLogger. # we know. # - after startup, it continues to monitor the dbus: # 1) when services are added we do the same check on that # 2) when services are removed, we remove any items that we had that referred to that service # 3) if an existing services adds paths we update ourselves as well: on init, we make a # VeDbusItemImport for a non-, or not yet existing objectpaths as well1 # # Code is used by the vrmLogger, and also the pubsub code. Both are other modules in the dbus_vrm repo. from dbus.mainloop.glib import DBusGMainLoop from gi.repository import GLib import dbus import dbus.service import inspect import logging import argparse import pprint import traceback import os from collections import defaultdict from functools import partial # our own packages from ve_utils import exit_on_error, wrap_dbus_value, unwrap_dbus_value notfound = object() # For lookups where None is a valid result logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) class SystemBus(dbus.bus.BusConnection): def __new__(cls): return dbus.bus.BusConnection.__new__(cls, dbus.bus.BusConnection.TYPE_SYSTEM) class SessionBus(dbus.bus.BusConnection): def __new__(cls): return dbus.bus.BusConnection.__new__(cls, dbus.bus.BusConnection.TYPE_SESSION) class MonitoredValue(object): def __init__(self, value, text, options): super(MonitoredValue, self).__init__() self.value = value self.text = text self.options = options # For legacy code, allow treating this as a tuple/list def __iter__(self): return iter((self.value, self.text, self.options)) class Service(object): def __init__(self, id, serviceName, deviceInstance): super(Service, self).__init__() self.id = id self.name = serviceName self.paths = {} self._seen = set() self.deviceInstance = deviceInstance # For legacy code, attributes can still be accessed as if keys from a # dictionary. def __setitem__(self, key, value): self.__dict__[key] = value def __getitem__(self, key): return self.__dict__[key] def set_seen(self, path): self._seen.add(path) def seen(self, path): return path in self._seen @property def service_class(self): return '.'.join(self.name.split('.')[:3]) class DbusMonitor(object): ## Constructor def __init__(self, dbusTree, valueChangedCallback=None, deviceAddedCallback=None, deviceRemovedCallback=None, namespace="com.victronenergy"): # valueChangedCallback is the callback that we call when something has changed. # def value_changed_on_dbus(dbusServiceName, dbusPath, options, changes, deviceInstance): # in which changes is a tuple with GetText() and GetValue() self.valueChangedCallback = valueChangedCallback self.deviceAddedCallback = deviceAddedCallback self.deviceRemovedCallback = deviceRemovedCallback self.dbusTree = dbusTree # Lists all tracked services. Stores name, id, device instance, value per path, and whenToLog info # indexed by service name (eg. com.victronenergy.settings). self.servicesByName = {} # Same values as self.servicesByName, but indexed by service id (eg. :1.30) self.servicesById = {} # Keep track of services by class to speed up calls to get_service_list self.servicesByClass = defaultdict(list) # Keep track of any additional watches placed on items self.serviceWatches = defaultdict(list) # For a PC, connect to the SessionBus # For a CCGX, connect to the SystemBus self.dbusConn = SessionBus() if 'DBUS_SESSION_BUS_ADDRESS' in os.environ else SystemBus() # subscribe to NameOwnerChange for bus connect / disconnect events. # NOTE: this is on a different bus then the one above! standardBus = (dbus.SessionBus() if 'DBUS_SESSION_BUS_ADDRESS' in os.environ \ else dbus.SystemBus()) self.add_name_owner_changed_receiver(standardBus, self.dbus_name_owner_changed) # Subscribe to PropertiesChanged for all services self.dbusConn.add_signal_receiver(self.handler_value_changes, dbus_interface='com.victronenergy.BusItem', signal_name='PropertiesChanged', path_keyword='path', sender_keyword='senderId') # Subscribe to ItemsChanged for all services self.dbusConn.add_signal_receiver(self.handler_item_changes, dbus_interface='com.victronenergy.BusItem', signal_name='ItemsChanged', path='/', sender_keyword='senderId') logger.info('===== Search on dbus for services that we will monitor starting... =====') serviceNames = self.dbusConn.list_names() for serviceName in serviceNames: self.scan_dbus_service(serviceName) logger.info('===== Search on dbus for services that we will monitor finished =====') @staticmethod def make_service(serviceId, serviceName, deviceInstance): """ Override this to use a different kind of service object. """ return Service(serviceId, serviceName, deviceInstance) def make_monitor(self, service, path, value, text, options): """ Override this to do more things with monitoring. """ return MonitoredValue(unwrap_dbus_value(value), unwrap_dbus_value(text), options) def dbus_name_owner_changed(self, name, oldowner, newowner): if not name.startswith("com.victronenergy."): return #decouple, and process in main loop GLib.idle_add(exit_on_error, self._process_name_owner_changed, name, oldowner, newowner) @staticmethod # When supported, only name owner changes for the the given namespace are reported. This # prevents spending cpu time at irrelevant changes, like scripts accessing the bus temporarily. def add_name_owner_changed_receiver(dbus, name_owner_changed, namespace="com.victronenergy"): # support for arg0namespace is submitted upstream, but not included at the time of # writing, Venus OS does support it, so try if it works. if namespace is None: dbus.add_signal_receiver(name_owner_changed, signal_name='NameOwnerChanged') else: try: dbus.add_signal_receiver(name_owner_changed, signal_name='NameOwnerChanged', arg0namespace=namespace) except TypeError: dbus.add_signal_receiver(name_owner_changed, signal_name='NameOwnerChanged') def _process_name_owner_changed(self, name, oldowner, newowner): if newowner != '': # so we found some new service. Check if we can do something with it. newdeviceadded = self.scan_dbus_service(name) if newdeviceadded and self.deviceAddedCallback is not None: self.deviceAddedCallback(name, self.get_device_instance(name)) elif name in self.servicesByName: # it disappeared, we need to remove it. logger.info("%s disappeared from the dbus. Removing it from our lists" % name) service = self.servicesByName[name] del self.servicesById[service.id] del self.servicesByName[name] for watch in self.serviceWatches[name]: watch.remove() del self.serviceWatches[name] self.servicesByClass[service.service_class].remove(service) if self.deviceRemovedCallback is not None: self.deviceRemovedCallback(name, service.deviceInstance) def scan_dbus_service(self, serviceName): try: return self.scan_dbus_service_inner(serviceName) except: logger.error("Ignoring %s because of error while scanning:" % (serviceName)) traceback.print_exc() return False # Errors 'org.freedesktop.DBus.Error.ServiceUnknown' and # 'org.freedesktop.DBus.Error.Disconnected' seem to happen when the service # disappears while its being scanned. Which might happen, but is not really # normal either, so letting them go into the logs. # Scans the given dbus service to see if it contains anything interesting for us. If it does, add # it to our list of monitored D-Bus services. def scan_dbus_service_inner(self, serviceName): # make it a normal string instead of dbus string serviceName = str(serviceName) paths = self.dbusTree.get('.'.join(serviceName.split('.')[0:3]), None) if paths is None: logger.debug("Ignoring service %s, not in the tree" % serviceName) return False logger.info("Found: %s, scanning and storing items" % serviceName) serviceId = self.dbusConn.get_name_owner(serviceName) # we should never be notified to add a D-Bus service that we already have. If this assertion # raises, check process_name_owner_changed, and D-Bus workings. assert serviceName not in self.servicesByName assert serviceId not in self.servicesById if serviceName == 'com.victronenergy.settings': di = 0 elif serviceName.startswith('com.victronenergy.vecan.'): di = 0 else: try: di = self.dbusConn.call_blocking(serviceName, '/DeviceInstance', None, 'GetValue', '', []) except dbus.exceptions.DBusException: logger.info(" %s was skipped because it has no device instance" % serviceName) return False # Skip it else: di = int(di) logger.info(" %s has device instance %s" % (serviceName, di)) service = self.make_service(serviceId, serviceName, di) # Let's try to fetch everything in one go values = {} texts = {} try: values.update(self.dbusConn.call_blocking(serviceName, '/', None, 'GetValue', '', [])) texts.update(self.dbusConn.call_blocking(serviceName, '/', None, 'GetText', '', [])) except: pass for path, options in paths.items(): # path will be the D-Bus path: '/Ac/ActiveIn/L1/V' # options will be a dictionary: {'code': 'V', 'whenToLog': 'onIntervalAlways'} # Try to obtain the value we want from our bulk fetch. If we # cannot find it there, do an individual query. value = values.get(path[1:], notfound) if value != notfound: service.set_seen(path) text = texts.get(path[1:], notfound) if value is notfound or text is notfound: try: value = self.dbusConn.call_blocking(serviceName, path, None, 'GetValue', '', []) service.set_seen(path) text = self.dbusConn.call_blocking(serviceName, path, None, 'GetText', '', []) except dbus.exceptions.DBusException as e: if e.get_dbus_name() in ( 'org.freedesktop.DBus.Error.ServiceUnknown', 'org.freedesktop.DBus.Error.Disconnected'): raise # This exception will be handled below # TODO org.freedesktop.DBus.Error.UnknownMethod really # shouldn't happen but sometimes does. logger.debug("%s %s does not exist (yet)" % (serviceName, path)) value = None text = None service.paths[path] = self.make_monitor(service, path, unwrap_dbus_value(value), unwrap_dbus_value(text), options) logger.debug("Finished scanning and storing items for %s" % serviceName) # Adjust self at the end of the scan, so we don't have an incomplete set of # data if an exception occurs during the scan. self.servicesByName[serviceName] = service self.servicesById[serviceId] = service self.servicesByClass[service.service_class].append(service) return True def handler_item_changes(self, items, senderId): if not isinstance(items, dict): return try: service = self.servicesById[senderId] except KeyError: # senderId isn't there, which means it hasn't been scanned yet. return for path, changes in items.items(): try: v = unwrap_dbus_value(changes['Value']) except (KeyError, TypeError): continue try: t = changes['Text'] except KeyError: t = str(v) self._handler_value_changes(service, path, v, t) def handler_value_changes(self, changes, path, senderId): # If this properyChange does not involve a value, our work is done. if 'Value' not in changes: return try: service = self.servicesById[senderId] except KeyError: # senderId isn't there, which means it hasn't been scanned yet. return v = unwrap_dbus_value(changes['Value']) # Some services don't send Text with their PropertiesChanged events. try: t = changes['Text'] except KeyError: t = str(v) self._handler_value_changes(service, path, v, t) def _handler_value_changes(self, service, path, value, text): try: a = service.paths[path] except KeyError: # path isn't there, which means it hasn't been scanned yet. return service.set_seen(path) # First update our store to the new value if a.value == value: return a.value = value a.text = text # And do the rest of the processing in on the mainloop if self.valueChangedCallback is not None: GLib.idle_add(exit_on_error, self._execute_value_changes, service.name, path, { 'Value': value, 'Text': text}, a.options) def _execute_value_changes(self, serviceName, objectPath, changes, options): # double check that the service still exists, as it might have # disappeared between scheduling-for and executing this function. if serviceName not in self.servicesByName: return self.valueChangedCallback(serviceName, objectPath, options, changes, self.get_device_instance(serviceName)) # Gets the value for a certain servicename and path # The default_value is returned when: # 1. When the service doesn't exist. # 2. When the path asked for isn't being monitored. # 3. When the path exists, but has dbus-invalid, ie an empty byte array. # 4. When the path asked for is being monitored, but doesn't exist for that service. def get_value(self, serviceName, objectPath, default_value=None): service = self.servicesByName.get(serviceName, None) if service is None: return default_value value = service.paths.get(objectPath, None) if value is None or value.value is None: return default_value return value.value # returns if a dbus exists now, by doing a blocking dbus call. # Typically seen will be sufficient and doesn't need access to the dbus. def exists(self, serviceName, objectPath): try: self.dbusConn.call_blocking(serviceName, objectPath, None, 'GetValue', '', []) return True except dbus.exceptions.DBusException as e: return False # Returns if there ever was a successful GetValue or valueChanged event. # Unlike get_value this return True also if the actual value is invalid. # # Note: the path might no longer exists anymore, but that doesn't happen in # practice. If a service really wants to reconfigure itself typically it should # reconnect to the dbus which causes it to be rescanned and seen will be updated. # If it is really needed to know if a path still exists, use exists. def seen(self, serviceName, objectPath): try: return self.servicesByName[serviceName].seen(objectPath) except KeyError: return False # Sets the value for a certain servicename and path, returns the return value of the D-Bus SetValue # method. If the underlying item does not exist (the service does not exist, or the objectPath was not # registered) the function will return -1 def set_value(self, serviceName, objectPath, value): # Check if the D-Bus object referenced by serviceName and objectPath is registered. There is no # necessity to do this, but it is in line with previous implementations which kept VeDbusItemImport # objects for registers items only. service = self.servicesByName.get(serviceName, None) if service is None: return -1 if objectPath not in service.paths: return -1 # We do not catch D-Bus exceptions here, because the previous implementation did not do that either. return self.dbusConn.call_blocking(serviceName, objectPath, dbus_interface='com.victronenergy.BusItem', method='SetValue', signature=None, args=[wrap_dbus_value(value)]) # Similar to set_value, but operates asynchronously def set_value_async(self, serviceName, objectPath, value, reply_handler=None, error_handler=None): service = self.servicesByName.get(serviceName, None) if service is not None: if objectPath in service.paths: self.dbusConn.call_async(serviceName, objectPath, dbus_interface='com.victronenergy.BusItem', method='SetValue', signature=None, args=[wrap_dbus_value(value)], reply_handler=reply_handler, error_handler=error_handler) return if error_handler is not None: error_handler(TypeError('Service or path not found, ' 'service=%s, path=%s' % (serviceName, objectPath))) # returns a dictionary, keys are the servicenames, value the instances # optionally use the classfilter to get only a certain type of services, for # example com.victronenergy.battery. def get_service_list(self, classfilter=None): if classfilter is None: return { servicename: service.deviceInstance \ for servicename, service in self.servicesByName.items() } if classfilter not in self.servicesByClass: return {} return { service.name: service.deviceInstance \ for service in self.servicesByClass[classfilter] } def get_device_instance(self, serviceName): return self.servicesByName[serviceName].deviceInstance def track_value(self, serviceName, objectPath, callback, *args, **kwargs): """ A DbusMonitor can watch specific service/path combos for changes so that it is not fully reliant on the global handler_value_changes in this class. Additional watches are deleted automatically when the service disappears from dbus. """ cb = partial(callback, *args, **kwargs) def root_tracker(items): # Check if objectPath in dict try: v = items[objectPath] _v = unwrap_dbus_value(v['Value']) except (KeyError, TypeError): return # not in this dict try: t = v['Text'] except KeyError: cb({'Value': _v }) else: cb({'Value': _v, 'Text': t}) # Track changes on the path, and also on root self.serviceWatches[serviceName].extend(( self.dbusConn.add_signal_receiver(cb, dbus_interface='com.victronenergy.BusItem', signal_name='PropertiesChanged', path=objectPath, bus_name=serviceName), self.dbusConn.add_signal_receiver(root_tracker, dbus_interface='com.victronenergy.BusItem', signal_name='ItemsChanged', path="/", bus_name=serviceName), )) # ====== ALL CODE BELOW THIS LINE IS PURELY FOR DEVELOPING THIS CLASS ====== # Example function that can be used as a starting point to use this code def value_changed_on_dbus(dbusServiceName, dbusPath, dict, changes, deviceInstance): logger.debug("0 ----------------") logger.debug("1 %s%s changed" % (dbusServiceName, dbusPath)) logger.debug("2 vrm dict : %s" % dict) logger.debug("3 changes-text: %s" % changes['Text']) logger.debug("4 changes-value: %s" % changes['Value']) logger.debug("5 deviceInstance: %s" % deviceInstance) logger.debug("6 - end") def nameownerchange(a, b): # used to find memory leaks in dbusmonitor and VeDbusItemImport import gc gc.collect() objects = gc.get_objects() print (len([o for o in objects if type(o).__name__ == 'VeDbusItemImport'])) print (len([o for o in objects if type(o).__name__ == 'SignalMatch'])) print (len(objects)) def print_values(dbusmonitor): a = dbusmonitor.get_value('wrongservice', '/DbusInvalid', default_value=1000) b = dbusmonitor.get_value('com.victronenergy.dummyservice.ttyO1', '/NotInTheMonitorList', default_value=1000) c = dbusmonitor.get_value('com.victronenergy.dummyservice.ttyO1', '/DbusInvalid', default_value=1000) d = dbusmonitor.get_value('com.victronenergy.dummyservice.ttyO1', '/NonExistingButMonitored', default_value=1000) print ("All should be 1000: Wrong Service: %s, NotInTheMonitorList: %s, DbusInvalid: %s, NonExistingButMonitored: %s" % (a, b, c, d)) return True # We have a mainloop, but that is just for developing this code. Normally above class & code is used from # some other class, such as vrmLogger or the pubsub Implementation. def main(): # Init logging logging.basicConfig(level=logging.DEBUG) logger.info(__file__ + " is starting up") # Have a mainloop, so we can send/receive asynchronous calls to and from dbus DBusGMainLoop(set_as_default=True) import os import sys sys.path.insert(1, os.path.join(os.path.dirname(__file__), '../../')) dummy = {'code': None, 'whenToLog': 'configChange', 'accessLevel': None} monitorlist = {'com.victronenergy.dummyservice': { '/Connected': dummy, '/ProductName': dummy, '/Mgmt/Connection': dummy, '/Dc/0/Voltage': dummy, '/Dc/0/Current': dummy, '/Dc/0/Temperature': dummy, '/Load/I': dummy, '/FirmwareVersion': dummy, '/DbusInvalid': dummy, '/NonExistingButMonitored': dummy}} d = DbusMonitor(monitorlist, value_changed_on_dbus, deviceAddedCallback=nameownerchange, deviceRemovedCallback=nameownerchange) GLib.timeout_add(1000, print_values, d) # Start and run the mainloop logger.info("Starting mainloop, responding on only events") mainloop = GLib.MainLoop() mainloop.run() if __name__ == "__main__": main() ================================================ FILE: velib_python/velib_python/v3.34/oldestVersion ================================================ v3.10 ================================================ FILE: velib_python/velib_python/v3.34/settingsdevice.py ================================================ import dbus import logging import time from functools import partial # Local imports from vedbus import VeDbusItemImport ## Indexes for the setting dictonary. PATH = 0 VALUE = 1 MINIMUM = 2 MAXIMUM = 3 SILENT = 4 ## The Settings Device class. # Used by python programs, such as the vrm-logger, to read and write settings they # need to store on disk. And since these settings might be changed from a different # source, such as the GUI, the program can pass an eventCallback that will be called # as soon as some setting is changed. # # The settings are stored in flash via the com.victronenergy.settings service on dbus. # See https://github.com/victronenergy/localsettings for more info. # # If there are settings in de supportSettings list which are not yet on the dbus, # and therefore not yet in the xml file, they will be added through the dbus-addSetting # interface of com.victronenergy.settings. class SettingsDevice(object): ## The constructor processes the tree of dbus-items. # @param bus the system-dbus object # @param name the dbus-service-name of the settings dbus service, 'com.victronenergy.settings' # @param supportedSettings dictionary with all setting-names, and their defaultvalue, min, max and whether # the setting is silent. The 'silent' entry is optional. If set to true, no changes in the setting will # be logged by localsettings. # @param eventCallback function that will be called on changes on any of these settings # @param timeout Maximum interval to wait for localsettings. An exception is thrown at the end of the # interval if the localsettings D-Bus service has not appeared yet. def __init__(self, bus, supportedSettings, eventCallback, name='com.victronenergy.settings', timeout=0): logging.debug("===== Settings device init starting... =====") self._bus = bus self._dbus_name = name self._eventCallback = eventCallback self._values = {} # stored the values, used to pass the old value along on a setting change self._settings = {} count = 0 while True: if 'com.victronenergy.settings' in self._bus.list_names(): break if count == timeout: raise Exception("The settings service com.victronenergy.settings does not exist!") count += 1 logging.info('waiting for settings') time.sleep(1) # Add the items. self.addSettings(supportedSettings) logging.debug("===== Settings device init finished =====") def addSettings(self, settings): for setting, options in settings.items(): silent = len(options) > SILENT and options[SILENT] busitem = self.addSetting(options[PATH], options[VALUE], options[MINIMUM], options[MAXIMUM], silent, callback=partial(self.handleChangedSetting, setting)) self._settings[setting] = busitem self._values[setting] = busitem.get_value() def addSetting(self, path, value, _min, _max, silent=False, callback=None): busitem = VeDbusItemImport(self._bus, self._dbus_name, path, callback) if busitem.exists and (value, _min, _max, silent) == busitem._proxy.GetAttributes(): logging.debug("Setting %s found" % path) else: logging.info("Setting %s does not exist yet or must be adjusted" % path) # Prepare to add the setting. Most dbus types extend the python # type so it is only necessary to additionally test for Int64. if isinstance(value, (int, dbus.Int64)): itemType = 'i' elif isinstance(value, float): itemType = 'f' else: itemType = 's' # Add the setting # TODO, make an object that inherits VeDbusItemImport, and complete the D-Bus settingsitem interface settings_item = VeDbusItemImport(self._bus, self._dbus_name, '/Settings', createsignal=False) setting_path = path.replace('/Settings/', '', 1) if silent: settings_item._proxy.AddSilentSetting('', setting_path, value, itemType, _min, _max) else: settings_item._proxy.AddSetting('', setting_path, value, itemType, _min, _max) busitem = VeDbusItemImport(self._bus, self._dbus_name, path, callback) return busitem def handleChangedSetting(self, setting, servicename, path, changes): oldvalue = self._values[setting] if setting in self._values else None self._values[setting] = changes['Value'] if self._eventCallback is None: return self._eventCallback(setting, oldvalue, changes['Value']) def setDefault(self, path): item = VeDbusItemImport(self._bus, self._dbus_name, path, createsignal=False) item.set_default() def __getitem__(self, setting): return self._settings[setting].get_value() def __setitem__(self, setting, newvalue): result = self._settings[setting].set_value(newvalue) if result != 0: # Trying to make some false change to our own settings? How dumb! assert False ================================================ FILE: velib_python/velib_python/v3.34/ve_utils.py ================================================ #!/usr/bin/env python3 # -*- coding: utf-8 -*- import sys from traceback import print_exc from os import _exit as os_exit from os import statvfs from subprocess import check_output, CalledProcessError import logging import dbus logger = logging.getLogger(__name__) VEDBUS_INVALID = dbus.Array([], signature=dbus.Signature('i'), variant_level=1) class NoVrmPortalIdError(Exception): pass # Use this function to make sure the code quits on an unexpected exception. Make sure to use it # when using GLib.idle_add and also GLib.timeout_add. # Without this, the code will just keep running, since GLib does not stop the mainloop on an # exception. # Example: GLib.idle_add(exit_on_error, myfunc, arg1, arg2) def exit_on_error(func, *args, **kwargs): try: return func(*args, **kwargs) except: try: print ('exit_on_error: there was an exception. Printing stacktrace will be tried and then exit') print_exc() except: pass # sys.exit() is not used, since that throws an exception, which does not lead to a program # halt when used in a dbus callback, see connection.py in the Python/Dbus libraries, line 230. os_exit(1) __vrm_portal_id = None def get_vrm_portal_id(): # The original definition of the VRM Portal ID is that it is the mac # address of the onboard- ethernet port (eth0), stripped from its colons # (:) and lower case. This may however differ between platforms. On Venus # the task is therefore deferred to /sbin/get-unique-id so that a # platform specific method can be easily defined. # # If /sbin/get-unique-id does not exist, then use the ethernet address # of eth0. This also handles the case where velib_python is used as a # package install on a Raspberry Pi. # # On a Linux host where the network interface may not be eth0, you can set # the VRM_IFACE environment variable to the correct name. global __vrm_portal_id if __vrm_portal_id: return __vrm_portal_id portal_id = None # First try the method that works if we don't have a data partition. This # will fail when the current user is not root. try: portal_id = check_output("/sbin/get-unique-id").decode("utf-8", "ignore").strip() if not portal_id: raise NoVrmPortalIdError("get-unique-id returned blank") __vrm_portal_id = portal_id return portal_id except CalledProcessError: # get-unique-id returned non-zero raise NoVrmPortalIdError("get-unique-id returned non-zero") except OSError: # File doesn't exist, use fallback pass # Fall back to getting our id using a syscall. Assume we are on linux. # Allow the user to override what interface is used using an environment # variable. import fcntl, socket, struct, os iface = os.environ.get('VRM_IFACE', 'eth0').encode('ascii') s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) try: info = fcntl.ioctl(s.fileno(), 0x8927, struct.pack('256s', iface[:15])) except IOError: raise NoVrmPortalIdError("ioctl failed for eth0") __vrm_portal_id = info[18:24].hex() return __vrm_portal_id # See VE.Can registers - public.docx for definition of this conversion def convert_vreg_version_to_readable(version): def str_to_arr(x, length): a = [] for i in range(0, len(x), length): a.append(x[i:i+length]) return a x = "%x" % version x = x.upper() if len(x) == 5 or len(x) == 3 or len(x) == 1: x = '0' + x a = str_to_arr(x, 2); # remove the first 00 if there are three bytes and it is 00 if len(a) == 3 and a[0] == '00': a.remove(0); # if we have two or three bytes now, and the first character is a 0, remove it if len(a) >= 2 and a[0][0:1] == '0': a[0] = a[0][1]; result = '' for item in a: result += ('.' if result != '' else '') + item result = 'v' + result return result def get_free_space(path): result = -1 try: s = statvfs(path) result = s.f_frsize * s.f_bavail # Number of free bytes that ordinary users except Exception as ex: logger.info("Error while retrieving free space for path %s: %s" % (path, ex)) return result def _get_sysfs_machine_name(): try: with open('/sys/firmware/devicetree/base/model', 'r') as f: return f.read().rstrip('\x00') except IOError: pass return None # Returns None if it cannot find a machine name. Otherwise returns the string # containing the name def get_machine_name(): # First try calling the venus utility script try: return check_output("/usr/bin/product-name").strip().decode('UTF-8') except (CalledProcessError, OSError): pass # Fall back to sysfs name = _get_sysfs_machine_name() if name is not None: return name # Fall back to venus build machine name try: with open('/etc/venus/machine', 'r', encoding='UTF-8') as f: return f.read().strip() except IOError: pass return None def get_product_id(): """ Find the machine ID and return it. """ # First try calling the venus utility script try: return check_output("/usr/bin/product-id").strip().decode('UTF-8') except (CalledProcessError, OSError): pass # Fall back machine name mechanism name = _get_sysfs_machine_name() return { 'Color Control GX': 'C001', 'Venus GX': 'C002', 'Octo GX': 'C006', 'EasySolar-II': 'C007', 'MultiPlus-II': 'C008', 'Maxi GX': 'C009', 'Cerbo GX': 'C00A' }.get(name, 'C003') # C003 is Generic # Returns False if it cannot open the file. Otherwise returns its rstripped contents def read_file(path): content = False try: with open(path, 'r') as f: content = f.read().rstrip() except Exception as ex: logger.debug("Error while reading %s: %s" % (path, ex)) return content def wrap_dbus_value(value): if value is None: return VEDBUS_INVALID if isinstance(value, float): return dbus.Double(value, variant_level=1) if isinstance(value, bool): return dbus.Boolean(value, variant_level=1) if isinstance(value, int): try: return dbus.Int32(value, variant_level=1) except OverflowError: return dbus.Int64(value, variant_level=1) if isinstance(value, str): return dbus.String(value, variant_level=1) if isinstance(value, list): if len(value) == 0: # If the list is empty we cannot infer the type of the contents. So assume unsigned integer. # A (signed) integer is dangerous, because an empty list of signed integers is used to encode # an invalid value. return dbus.Array([], signature=dbus.Signature('u'), variant_level=1) return dbus.Array([wrap_dbus_value(x) for x in value], variant_level=1) if isinstance(value, dict): # Wrapping the keys of the dictionary causes D-Bus errors like: # 'arguments to dbus_message_iter_open_container() were incorrect, # assertion "(type == DBUS_TYPE_ARRAY && contained_signature && # *contained_signature == DBUS_DICT_ENTRY_BEGIN_CHAR) || (contained_signature == NULL || # _dbus_check_is_valid_signature (contained_signature))" failed in file ...' return dbus.Dictionary({(k, wrap_dbus_value(v)) for k, v in value.items()}, variant_level=1) return value dbus_int_types = (dbus.Int32, dbus.UInt32, dbus.Byte, dbus.Int16, dbus.UInt16, dbus.UInt32, dbus.Int64, dbus.UInt64) def unwrap_dbus_value(val): """Converts D-Bus values back to the original type. For example if val is of type DBus.Double, a float will be returned.""" if isinstance(val, dbus_int_types): return int(val) if isinstance(val, dbus.Double): return float(val) if isinstance(val, dbus.Array): v = [unwrap_dbus_value(x) for x in val] return None if len(v) == 0 else v if isinstance(val, (dbus.Signature, dbus.String)): return str(val) # Python has no byte type, so we convert to an integer. if isinstance(val, dbus.Byte): return int(val) if isinstance(val, dbus.ByteArray): return "".join([bytes(x) for x in val]) if isinstance(val, (list, tuple)): return [unwrap_dbus_value(x) for x in val] if isinstance(val, (dbus.Dictionary, dict)): # Do not unwrap the keys, see comment in wrap_dbus_value return dict([(x, unwrap_dbus_value(y)) for x, y in val.items()]) if isinstance(val, dbus.Boolean): return bool(val) return val ================================================ FILE: velib_python/velib_python/v3.34/vedbus.py ================================================ #!/usr/bin/env python3 # -*- coding: utf-8 -*- import dbus.service import logging import traceback import os import weakref from collections import defaultdict from ve_utils import wrap_dbus_value, unwrap_dbus_value # vedbus contains three classes: # VeDbusItemImport -> use this to read data from the dbus, ie import # VeDbusItemExport -> use this to export data to the dbus (one value) # VeDbusService -> use that to create a service and export several values to the dbus # Code for VeDbusItemImport is copied from busitem.py and thereafter modified. # All projects that used busitem.py need to migrate to this package. And some # projects used to define there own equivalent of VeDbusItemExport. Better to # use VeDbusItemExport, or even better the VeDbusService class that does it all for you. # TODOS # 1 check for datatypes, it works now, but not sure if all is compliant with # com.victronenergy.BusItem interface definition. See also the files in # tests_and_examples. And see 'if type(v) == dbus.Byte:' on line 102. Perhaps # something similar should also be done in VeDbusBusItemExport? # 2 Shouldn't VeDbusBusItemExport inherit dbus.service.Object? # 7 Make hard rules for services exporting data to the D-Bus, in order to make tracking # changes possible. Does everybody first invalidate its data before leaving the bus? # And what about before taking one object away from the bus, instead of taking the # whole service offline? # They should! And after taking one value away, do we need to know that someone left # the bus? Or we just keep that value in invalidated for ever? Result is that we can't # see the difference anymore between an invalidated value and a value that was first on # the bus and later not anymore. See comments above VeDbusItemImport as well. # 9 there are probably more todos in the code below. # Some thoughts with regards to the data types: # # Text from: http://dbus.freedesktop.org/doc/dbus-python/doc/tutorial.html#data-types # --- # Variants are represented by setting the variant_level keyword argument in the # constructor of any D-Bus data type to a value greater than 0 (variant_level 1 # means a variant containing some other data type, variant_level 2 means a variant # containing a variant containing some other data type, and so on). If a non-variant # is passed as an argument but introspection indicates that a variant is expected, # it'll automatically be wrapped in a variant. # --- # # Also the different dbus datatypes, such as dbus.Int32, and dbus.UInt32 are a subclass # of Python int. dbus.String is a subclass of Python standard class unicode, etcetera # # So all together that explains why we don't need to explicitly convert back and forth # between the dbus datatypes and the standard python datatypes. Note that all datatypes # in python are objects. Even an int is an object. # The signature of a variant is 'v'. # Export ourselves as a D-Bus service. class VeDbusService(object): def __init__(self, servicename, bus=None): # dict containing the VeDbusItemExport objects, with their path as the key. self._dbusobjects = {} self._dbusnodes = {} self._ratelimiters = [] self._dbusname = None # dict containing the onchange callbacks, for each object. Object path is the key self._onchangecallbacks = {} # Connect to session bus whenever present, else use the system bus self._dbusconn = bus or (dbus.SessionBus() if 'DBUS_SESSION_BUS_ADDRESS' in os.environ else dbus.SystemBus()) # make the dbus connection available to outside, could make this a true property instead, but ach.. self.dbusconn = self._dbusconn # Register ourselves on the dbus, trigger an error if already in use (do_not_queue) self._dbusname = dbus.service.BusName(servicename, self._dbusconn, do_not_queue=True) # Add the root item that will return all items as a tree self._dbusnodes['/'] = VeDbusRootExport(self._dbusconn, '/', self) logging.info("registered ourselves on D-Bus as %s" % servicename) # To force immediate deregistering of this dbus service and all its object paths, explicitly # call __del__(). def __del__(self): for node in list(self._dbusnodes.values()): node.__del__() self._dbusnodes.clear() for item in list(self._dbusobjects.values()): item.__del__() self._dbusobjects.clear() if self._dbusname: self._dbusname.__del__() # Forces call to self._bus.release_name(self._name), see source code self._dbusname = None # @param callbackonchange function that will be called when this value is changed. First parameter will # be the path of the object, second the new value. This callback should return # True to accept the change, False to reject it. def add_path(self, path, value, description="", writeable=False, onchangecallback=None, gettextcallback=None, valuetype=None): if onchangecallback is not None: self._onchangecallbacks[path] = onchangecallback item = VeDbusItemExport( self._dbusconn, path, value, description, writeable, self._value_changed, gettextcallback, deletecallback=self._item_deleted, valuetype=valuetype) spl = path.split('/') for i in range(2, len(spl)): subPath = '/'.join(spl[:i]) if subPath not in self._dbusnodes and subPath not in self._dbusobjects: self._dbusnodes[subPath] = VeDbusTreeExport(self._dbusconn, subPath, self) self._dbusobjects[path] = item logging.debug('added %s with start value %s. Writeable is %s' % (path, value, writeable)) # Add the mandatory paths, as per victron dbus api doc def add_mandatory_paths(self, processname, processversion, connection, deviceinstance, productid, productname, firmwareversion, hardwareversion, connected): self.add_path('/Mgmt/ProcessName', processname) self.add_path('/Mgmt/ProcessVersion', processversion) self.add_path('/Mgmt/Connection', connection) # Create rest of the mandatory objects self.add_path('/DeviceInstance', deviceinstance) self.add_path('/ProductId', productid) self.add_path('/ProductName', productname) self.add_path('/FirmwareVersion', firmwareversion) self.add_path('/HardwareVersion', hardwareversion) self.add_path('/Connected', connected) # Callback function that is called from the VeDbusItemExport objects when a value changes. This function # maps the change-request to the onchangecallback given to us for this specific path. def _value_changed(self, path, newvalue): if path not in self._onchangecallbacks: return True return self._onchangecallbacks[path](path, newvalue) def _item_deleted(self, path): self._dbusobjects.pop(path) for np in list(self._dbusnodes.keys()): if np != '/': for ip in self._dbusobjects: if ip.startswith(np + '/'): break else: self._dbusnodes[np].__del__() self._dbusnodes.pop(np) def __getitem__(self, path): return self._dbusobjects[path].local_get_value() def __setitem__(self, path, newvalue): self._dbusobjects[path].local_set_value(newvalue) def __delitem__(self, path): self._dbusobjects[path].__del__() # Invalidates and then removes the object path assert path not in self._dbusobjects def __contains__(self, path): return path in self._dbusobjects def __enter__(self): l = ServiceContext(self) self._ratelimiters.append(l) return l def __exit__(self, *exc): # pop off the top one and flush it. If with statements are nested # then each exit flushes its own part. if self._ratelimiters: self._ratelimiters.pop().flush() class ServiceContext(object): def __init__(self, parent): self.parent = parent self.changes = {} def __getitem__(self, path): return self.parent[path] def __setitem__(self, path, newvalue): c = self.parent._dbusobjects[path]._local_set_value(newvalue) if c is not None: self.changes[path] = c def flush(self): if self.changes: self.parent._dbusnodes['/'].ItemsChanged(self.changes) class TrackerDict(defaultdict): """ Same as defaultdict, but passes the key to default_factory. """ def __missing__(self, key): self[key] = x = self.default_factory(key) return x class VeDbusRootTracker(object): """ This tracks the root of a dbus path and listens for PropertiesChanged signals. When a signal arrives, parse it and unpack the key/value changes into traditional events, then pass it to the original eventCallback method. """ def __init__(self, bus, serviceName): self.importers = defaultdict(weakref.WeakSet) self.serviceName = serviceName self._match = bus.get_object(serviceName, '/', introspect=False).connect_to_signal( "ItemsChanged", weak_functor(self._items_changed_handler)) def __del__(self): self._match.remove() self._match = None def add(self, i): self.importers[i.path].add(i) def _items_changed_handler(self, items): if not isinstance(items, dict): return for path, changes in items.items(): try: v = changes['Value'] except KeyError: continue try: t = changes['Text'] except KeyError: t = str(unwrap_dbus_value(v)) for i in self.importers.get(path, ()): i._properties_changed_handler({'Value': v, 'Text': t}) """ Importing basics: - If when we power up, the D-Bus service does not exist, or it does exist and the path does not yet exist, still subscribe to a signal: as soon as it comes online it will send a signal with its initial value, which VeDbusItemImport will receive and use to update local cache. And, when set, call the eventCallback. - If when we power up, save it - When using get_value, know that there is no difference between services (or object paths) that don't exist and paths that are invalid (= empty array, see above). Both will return None. In case you do really want to know ifa path exists or not, use the exists property. - When a D-Bus service leaves the D-Bus, it will first invalidate all its values, and send signals with that update, and only then leave the D-Bus. (or do we need to subscribe to the NameOwnerChanged- signal!?!) To be discussed and make sure. Not really urgent, since all existing code that uses this class already subscribes to the NameOwnerChanged signal, and subsequently removes instances of this class. Read when using this class: Note that when a service leaves that D-Bus without invalidating all its exported objects first, for example because it is killed, VeDbusItemImport doesn't have a clue. So when using VeDbusItemImport, make sure to also subscribe to the NamerOwnerChanged signal on bus-level. Or just use dbusmonitor, because that takes care of all of that for you. """ class VeDbusItemImport(object): def __new__(cls, bus, serviceName, path, eventCallback=None, createsignal=True): instance = object.__new__(cls) # If signal tracking should be done, also add to root tracker if createsignal: if "_roots" not in cls.__dict__: cls._roots = TrackerDict(lambda k: VeDbusRootTracker(bus, k)) return instance ## Constructor # @param bus the bus-object (SESSION or SYSTEM). # @param serviceName the dbus-service-name (string), for example 'com.victronenergy.battery.ttyO1' # @param path the object-path, for example '/Dc/V' # @param eventCallback function that you want to be called on a value change # @param createSignal only set this to False if you use this function to one time read a value. When # leaving it to True, make sure to also subscribe to the NameOwnerChanged signal # elsewhere. See also note some 15 lines up. def __init__(self, bus, serviceName, path, eventCallback=None, createsignal=True): # TODO: is it necessary to store _serviceName and _path? Isn't it # stored in the bus_getobjectsomewhere? self._serviceName = serviceName self._path = path self._match = None # TODO: _proxy is being used in settingsdevice.py, make a getter for that self._proxy = bus.get_object(serviceName, path, introspect=False) self.eventCallback = eventCallback assert eventCallback is None or createsignal == True if createsignal: self._match = self._proxy.connect_to_signal( "PropertiesChanged", weak_functor(self._properties_changed_handler)) self._roots[serviceName].add(self) # store the current value in _cachedvalue. When it doesn't exists set _cachedvalue to # None, same as when a value is invalid self._cachedvalue = None try: v = self._proxy.GetValue() except dbus.exceptions.DBusException: pass else: self._cachedvalue = unwrap_dbus_value(v) def __del__(self): if self._match is not None: self._match.remove() self._match = None self._proxy = None def _refreshcachedvalue(self): self._cachedvalue = unwrap_dbus_value(self._proxy.GetValue()) ## Returns the path as a string, for example '/AC/L1/V' @property def path(self): return self._path ## Returns the dbus service name as a string, for example com.victronenergy.vebus.ttyO1 @property def serviceName(self): return self._serviceName ## Returns the value of the dbus-item. # the type will be a dbus variant, for example dbus.Int32(0, variant_level=1) # this is not a property to keep the name consistant with the com.victronenergy.busitem interface # returns None when the property is invalid def get_value(self): return self._cachedvalue ## Writes a new value to the dbus-item def set_value(self, newvalue): r = self._proxy.SetValue(wrap_dbus_value(newvalue)) # instead of just saving the value, go to the dbus and get it. So we have the right type etc. if r == 0: self._refreshcachedvalue() return r ## Resets the item to its default value def set_default(self): self._proxy.SetDefault() self._refreshcachedvalue() ## Returns the text representation of the value. # For example when the value is an enum/int GetText might return the string # belonging to that enum value. Another example, for a voltage, GetValue # would return a float, 12.0Volt, and GetText could return 12 VDC. # # Note that this depends on how the dbus-producer has implemented this. def get_text(self): return self._proxy.GetText() ## Returns true of object path exists, and false if it doesn't @property def exists(self): # TODO: do some real check instead of this crazy thing. r = False try: r = self._proxy.GetValue() r = True except dbus.exceptions.DBusException: pass return r ## callback for the trigger-event. # @param eventCallback the event-callback-function. @property def eventCallback(self): return self._eventCallback @eventCallback.setter def eventCallback(self, eventCallback): self._eventCallback = eventCallback ## Is called when the value of the imported bus-item changes. # Stores the new value in our local cache, and calls the eventCallback, if set. def _properties_changed_handler(self, changes): if "Value" in changes: changes['Value'] = unwrap_dbus_value(changes['Value']) self._cachedvalue = changes['Value'] if self._eventCallback: # The reason behind this try/except is to prevent errors silently ending up the an error # handler in the dbus code. try: self._eventCallback(self._serviceName, self._path, changes) except: traceback.print_exc() os._exit(1) # sys.exit() is not used, since that also throws an exception class VeDbusTreeExport(dbus.service.Object): def __init__(self, bus, objectPath, service): dbus.service.Object.__init__(self, bus, objectPath) self._service = service logging.debug("VeDbusTreeExport %s has been created" % objectPath) def __del__(self): # self._get_path() will raise an exception when retrieved after the call to .remove_from_connection, # so we need a copy. path = self._get_path() if path is None: return self.remove_from_connection() logging.debug("VeDbusTreeExport %s has been removed" % path) def _get_path(self): if len(self._locations) == 0: return None return self._locations[0][1] def _get_value_handler(self, path, get_text=False): logging.debug("_get_value_handler called for %s" % path) r = {} px = path if not px.endswith('/'): px += '/' for p, item in self._service._dbusobjects.items(): if p.startswith(px): v = item.GetText() if get_text else wrap_dbus_value(item.local_get_value()) r[p[len(px):]] = v logging.debug(r) return r @dbus.service.method('com.victronenergy.BusItem', out_signature='v') def GetValue(self): value = self._get_value_handler(self._get_path()) return dbus.Dictionary(value, signature=dbus.Signature('sv'), variant_level=1) @dbus.service.method('com.victronenergy.BusItem', out_signature='v') def GetText(self): return self._get_value_handler(self._get_path(), True) def local_get_value(self): return self._get_value_handler(self.path) class VeDbusRootExport(VeDbusTreeExport): @dbus.service.signal('com.victronenergy.BusItem', signature='a{sa{sv}}') def ItemsChanged(self, changes): pass @dbus.service.method('com.victronenergy.BusItem', out_signature='a{sa{sv}}') def GetItems(self): return { path: { 'Value': wrap_dbus_value(item.local_get_value()), 'Text': item.GetText() } for path, item in self._service._dbusobjects.items() } class VeDbusItemExport(dbus.service.Object): ## Constructor of VeDbusItemExport # # Use this object to export (publish), values on the dbus # Creates the dbus-object under the given dbus-service-name. # @param bus The dbus object. # @param objectPath The dbus-object-path. # @param value Value to initialize ourselves with, defaults to None which means Invalid # @param description String containing a description. Can be called over the dbus with GetDescription() # @param writeable what would this do!? :). # @param callback Function that will be called when someone else changes the value of this VeBusItem # over the dbus. First parameter passed to callback will be our path, second the new # value. This callback should return True to accept the change, False to reject it. def __init__(self, bus, objectPath, value=None, description=None, writeable=False, onchangecallback=None, gettextcallback=None, deletecallback=None, valuetype=None): dbus.service.Object.__init__(self, bus, objectPath) self._onchangecallback = onchangecallback self._gettextcallback = gettextcallback self._value = value self._description = description self._writeable = writeable self._deletecallback = deletecallback self._type = valuetype # To force immediate deregistering of this dbus object, explicitly call __del__(). def __del__(self): # self._get_path() will raise an exception when retrieved after the # call to .remove_from_connection, so we need a copy. path = self._get_path() if path == None: return if self._deletecallback is not None: self._deletecallback(path) self.remove_from_connection() logging.debug("VeDbusItemExport %s has been removed" % path) def _get_path(self): if len(self._locations) == 0: return None return self._locations[0][1] ## Sets the value. And in case the value is different from what it was, a signal # will be emitted to the dbus. This function is to be used in the python code that # is using this class to export values to the dbus. # set value to None to indicate that it is Invalid def local_set_value(self, newvalue): changes = self._local_set_value(newvalue) if changes is not None: self.PropertiesChanged(changes) def _local_set_value(self, newvalue): if self._value == newvalue: return None self._value = newvalue return { 'Value': wrap_dbus_value(newvalue), 'Text': self.GetText() } def local_get_value(self): return self._value # ==== ALL FUNCTIONS BELOW THIS LINE WILL BE CALLED BY OTHER PROCESSES OVER THE DBUS ==== ## Dbus exported method SetValue # Function is called over the D-Bus by other process. It will first check (via callback) if new # value is accepted. And it is, stores it and emits a changed-signal. # @param value The new value. # @return completion-code When successful a 0 is return, and when not a -1 is returned. @dbus.service.method('com.victronenergy.BusItem', in_signature='v', out_signature='i') def SetValue(self, newvalue): if not self._writeable: return 1 # NOT OK newvalue = unwrap_dbus_value(newvalue) # If value type is enforced, cast it. If the type can be coerced # python will do it for us. This allows ints to become floats, # or bools to become ints. Additionally also allow None, so that # a path may be invalidated. if self._type is not None and newvalue is not None: try: newvalue = self._type(newvalue) except (ValueError, TypeError): return 1 # NOT OK if newvalue == self._value: return 0 # OK # call the callback given to us, and check if new value is OK. if (self._onchangecallback is None or (self._onchangecallback is not None and self._onchangecallback(self.__dbus_object_path__, newvalue))): self.local_set_value(newvalue) return 0 # OK return 2 # NOT OK ## Dbus exported method GetDescription # # Returns the a description. # @param language A language code (e.g. ISO 639-1 en-US). # @param length Lenght of the language string. # @return description @dbus.service.method('com.victronenergy.BusItem', in_signature='si', out_signature='s') def GetDescription(self, language, length): return self._description if self._description is not None else 'No description given' ## Dbus exported method GetValue # Returns the value. # @return the value when valid, and otherwise an empty array @dbus.service.method('com.victronenergy.BusItem', out_signature='v') def GetValue(self): return wrap_dbus_value(self._value) ## Dbus exported method GetText # Returns the value as string of the dbus-object-path. # @return text A text-value. '---' when local value is invalid @dbus.service.method('com.victronenergy.BusItem', out_signature='s') def GetText(self): if self._value is None: return '---' # Default conversion from dbus.Byte will get you a character (so 'T' instead of '84'), so we # have to convert to int first. Note that if a dbus.Byte turns up here, it must have come from # the application itself, as all data from the D-Bus should have been unwrapped by now. if self._gettextcallback is None and type(self._value) == dbus.Byte: return str(int(self._value)) if self._gettextcallback is None and self.__dbus_object_path__ == '/ProductId': return "0x%X" % self._value if self._gettextcallback is None: return str(self._value) return self._gettextcallback(self.__dbus_object_path__, self._value) ## The signal that indicates that the value has changed. # Other processes connected to this BusItem object will have subscribed to the # event when they want to track our state. @dbus.service.signal('com.victronenergy.BusItem', signature='a{sv}') def PropertiesChanged(self, changes): pass ## This class behaves like a regular reference to a class method (eg. self.foo), but keeps a weak reference ## to the object which method is to be called. ## Use this object to break circular references. class weak_functor: def __init__(self, f): self._r = weakref.ref(f.__self__) self._f = weakref.ref(f.__func__) def __call__(self, *args, **kargs): r = self._r() f = self._f() if r == None or f == None: return f(r, *args, **kargs) ================================================ FILE: velib_python/velib_python/v3.41/dbusmonitor.py ================================================ #!/usr/bin/env python3 # -*- coding: utf-8 -*- ## @package dbus_vrm # This code takes care of the D-Bus interface (not all of below is implemented yet): # - on startup it scans the dbus for services we know. For each known service found, it searches for # objects/paths we know. Everything we find is stored in items{}, and an event is registered: if a # value changes weĺl be notified and can pass that on to our owner. For example the vrmLogger. # we know. # - after startup, it continues to monitor the dbus: # 1) when services are added we do the same check on that # 2) when services are removed, we remove any items that we had that referred to that service # 3) if an existing services adds paths we update ourselves as well: on init, we make a # VeDbusItemImport for a non-, or not yet existing objectpaths as well1 # # Code is used by the vrmLogger, and also the pubsub code. Both are other modules in the dbus_vrm repo. from dbus.mainloop.glib import DBusGMainLoop from gi.repository import GLib import dbus import dbus.service import inspect import logging import argparse import pprint import traceback import os from collections import defaultdict from functools import partial # our own packages from ve_utils import exit_on_error, wrap_dbus_value, unwrap_dbus_value, add_name_owner_changed_receiver notfound = object() # For lookups where None is a valid result logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) class SystemBus(dbus.bus.BusConnection): def __new__(cls): return dbus.bus.BusConnection.__new__(cls, dbus.bus.BusConnection.TYPE_SYSTEM) class SessionBus(dbus.bus.BusConnection): def __new__(cls): return dbus.bus.BusConnection.__new__(cls, dbus.bus.BusConnection.TYPE_SESSION) class MonitoredValue(object): def __init__(self, value, text, options): super(MonitoredValue, self).__init__() self.value = value self.text = text self.options = options # For legacy code, allow treating this as a tuple/list def __iter__(self): return iter((self.value, self.text, self.options)) class Service(object): def __init__(self, id, serviceName, deviceInstance): super(Service, self).__init__() self.id = id self.name = serviceName self.paths = {} self._seen = set() self.deviceInstance = deviceInstance # For legacy code, attributes can still be accessed as if keys from a # dictionary. def __setitem__(self, key, value): self.__dict__[key] = value def __getitem__(self, key): return self.__dict__[key] def set_seen(self, path): self._seen.add(path) def seen(self, path): return path in self._seen @property def service_class(self): return '.'.join(self.name.split('.')[:3]) class DbusMonitor(object): ## Constructor def __init__(self, dbusTree, valueChangedCallback=None, deviceAddedCallback=None, deviceRemovedCallback=None, namespace="com.victronenergy", ignoreServices=[]): # valueChangedCallback is the callback that we call when something has changed. # def value_changed_on_dbus(dbusServiceName, dbusPath, options, changes, deviceInstance): # in which changes is a tuple with GetText() and GetValue() self.valueChangedCallback = valueChangedCallback self.deviceAddedCallback = deviceAddedCallback self.deviceRemovedCallback = deviceRemovedCallback self.dbusTree = dbusTree self.ignoreServices = ignoreServices # Lists all tracked services. Stores name, id, device instance, value per path, and whenToLog info # indexed by service name (eg. com.victronenergy.settings). self.servicesByName = {} # Same values as self.servicesByName, but indexed by service id (eg. :1.30) self.servicesById = {} # Keep track of services by class to speed up calls to get_service_list self.servicesByClass = defaultdict(list) # Keep track of any additional watches placed on items self.serviceWatches = defaultdict(list) # For a PC, connect to the SessionBus # For a CCGX, connect to the SystemBus self.dbusConn = SessionBus() if 'DBUS_SESSION_BUS_ADDRESS' in os.environ else SystemBus() # subscribe to NameOwnerChange for bus connect / disconnect events. # NOTE: this is on a different bus then the one above! standardBus = (dbus.SessionBus() if 'DBUS_SESSION_BUS_ADDRESS' in os.environ \ else dbus.SystemBus()) add_name_owner_changed_receiver(standardBus, self.dbus_name_owner_changed) # Subscribe to PropertiesChanged for all services self.dbusConn.add_signal_receiver(self.handler_value_changes, dbus_interface='com.victronenergy.BusItem', signal_name='PropertiesChanged', path_keyword='path', sender_keyword='senderId') # Subscribe to ItemsChanged for all services self.dbusConn.add_signal_receiver(self.handler_item_changes, dbus_interface='com.victronenergy.BusItem', signal_name='ItemsChanged', path='/', sender_keyword='senderId') logger.info('===== Search on dbus for services that we will monitor starting... =====') serviceNames = self.dbusConn.list_names() for serviceName in serviceNames: self.scan_dbus_service(serviceName) logger.info('===== Search on dbus for services that we will monitor finished =====') @staticmethod def make_service(serviceId, serviceName, deviceInstance): """ Override this to use a different kind of service object. """ return Service(serviceId, serviceName, deviceInstance) def make_monitor(self, service, path, value, text, options): """ Override this to do more things with monitoring. """ return MonitoredValue(unwrap_dbus_value(value), unwrap_dbus_value(text), options) def dbus_name_owner_changed(self, name, oldowner, newowner): if not name.startswith("com.victronenergy."): return #decouple, and process in main loop GLib.idle_add(exit_on_error, self._process_name_owner_changed, name, oldowner, newowner) def _process_name_owner_changed(self, name, oldowner, newowner): if newowner != '': # so we found some new service. Check if we can do something with it. newdeviceadded = self.scan_dbus_service(name) if newdeviceadded and self.deviceAddedCallback is not None: self.deviceAddedCallback(name, self.get_device_instance(name)) elif name in self.servicesByName: # it disappeared, we need to remove it. logger.info("%s disappeared from the dbus. Removing it from our lists" % name) service = self.servicesByName[name] del self.servicesById[service.id] del self.servicesByName[name] for watch in self.serviceWatches[name]: watch.remove() del self.serviceWatches[name] self.servicesByClass[service.service_class].remove(service) if self.deviceRemovedCallback is not None: self.deviceRemovedCallback(name, service.deviceInstance) def scan_dbus_service(self, serviceName): try: return self.scan_dbus_service_inner(serviceName) except: logger.error("Ignoring %s because of error while scanning:" % (serviceName)) traceback.print_exc() return False # Errors 'org.freedesktop.DBus.Error.ServiceUnknown' and # 'org.freedesktop.DBus.Error.Disconnected' seem to happen when the service # disappears while its being scanned. Which might happen, but is not really # normal either, so letting them go into the logs. # Scans the given dbus service to see if it contains anything interesting for us. If it does, add # it to our list of monitored D-Bus services. def scan_dbus_service_inner(self, serviceName): # make it a normal string instead of dbus string serviceName = str(serviceName) if (len(self.ignoreServices) != 0 and any(serviceName.startswith(x) for x in self.ignoreServices)): logger.debug("Ignoring service %s" % serviceName) return False paths = self.dbusTree.get('.'.join(serviceName.split('.')[0:3]), None) if paths is None: logger.debug("Ignoring service %s, not in the tree" % serviceName) return False logger.info("Found: %s, scanning and storing items" % serviceName) serviceId = self.dbusConn.get_name_owner(serviceName) # we should never be notified to add a D-Bus service that we already have. If this assertion # raises, check process_name_owner_changed, and D-Bus workings. assert serviceName not in self.servicesByName assert serviceId not in self.servicesById # Try to fetch everything with a GetItems, then fall back to older # methods if that fails try: values = self.dbusConn.call_blocking(serviceName, '/', None, 'GetItems', '', []) except dbus.exceptions.DBusException: logger.info("GetItems failed, trying legacy methods") else: return self.scan_dbus_service_getitems_done(serviceName, serviceId, values) if serviceName == 'com.victronenergy.settings': di = 0 elif serviceName.startswith('com.victronenergy.vecan.'): di = 0 else: try: di = self.dbusConn.call_blocking(serviceName, '/DeviceInstance', None, 'GetValue', '', []) except dbus.exceptions.DBusException: logger.info(" %s was skipped because it has no device instance" % serviceName) return False # Skip it else: di = int(di) logger.info(" %s has device instance %s" % (serviceName, di)) service = self.make_service(serviceId, serviceName, di) # Let's try to fetch everything in one go values = {} texts = {} try: values.update(self.dbusConn.call_blocking(serviceName, '/', None, 'GetValue', '', [])) texts.update(self.dbusConn.call_blocking(serviceName, '/', None, 'GetText', '', [])) except: pass for path, options in paths.items(): # path will be the D-Bus path: '/Ac/ActiveIn/L1/V' # options will be a dictionary: {'code': 'V', 'whenToLog': 'onIntervalAlways'} # Try to obtain the value we want from our bulk fetch. If we # cannot find it there, do an individual query. value = values.get(path[1:], notfound) if value != notfound: service.set_seen(path) text = texts.get(path[1:], notfound) if value is notfound or text is notfound: try: value = self.dbusConn.call_blocking(serviceName, path, None, 'GetValue', '', []) service.set_seen(path) text = self.dbusConn.call_blocking(serviceName, path, None, 'GetText', '', []) except dbus.exceptions.DBusException as e: if e.get_dbus_name() in ( 'org.freedesktop.DBus.Error.ServiceUnknown', 'org.freedesktop.DBus.Error.Disconnected'): raise # This exception will be handled below # TODO org.freedesktop.DBus.Error.UnknownMethod really # shouldn't happen but sometimes does. logger.debug("%s %s does not exist (yet)" % (serviceName, path)) value = None text = None service.paths[path] = self.make_monitor(service, path, unwrap_dbus_value(value), unwrap_dbus_value(text), options) logger.debug("Finished scanning and storing items for %s" % serviceName) # Adjust self at the end of the scan, so we don't have an incomplete set of # data if an exception occurs during the scan. self.servicesByName[serviceName] = service self.servicesById[serviceId] = service self.servicesByClass[service.service_class].append(service) return True def scan_dbus_service_getitems_done(self, serviceName, serviceId, values): # Keeping these exceptions for legacy reasons if serviceName == 'com.victronenergy.settings': di = 0 elif serviceName.startswith('com.victronenergy.vecan.'): di = 0 else: try: di = values['/DeviceInstance']['Value'] except KeyError: logger.info(" %s was skipped because it has no device instance" % serviceName) return False else: di = int(di) logger.info(" %s has device instance %s" % (serviceName, di)) service = self.make_service(serviceId, serviceName, di) paths = self.dbusTree.get('.'.join(serviceName.split('.')[0:3]), {}) for path, options in paths.items(): item = values.get(path, notfound) if item is notfound: service.paths[path] = self.make_monitor(service, path, None, None, options) else: service.set_seen(path) value = item.get('Value', None) text = item.get('Text', None) service.paths[path] = self.make_monitor(service, path, unwrap_dbus_value(value), unwrap_dbus_value(text), options) self.servicesByName[serviceName] = service self.servicesById[serviceId] = service self.servicesByClass[service.service_class].append(service) return True def handler_item_changes(self, items, senderId): if not isinstance(items, dict): return try: service = self.servicesById[senderId] except KeyError: # senderId isn't there, which means it hasn't been scanned yet. return for path, changes in items.items(): try: v = unwrap_dbus_value(changes['Value']) except (KeyError, TypeError): continue try: t = changes['Text'] except KeyError: t = str(v) self._handler_value_changes(service, path, v, t) def handler_value_changes(self, changes, path, senderId): # If this properyChange does not involve a value, our work is done. if 'Value' not in changes: return try: service = self.servicesById[senderId] except KeyError: # senderId isn't there, which means it hasn't been scanned yet. return v = unwrap_dbus_value(changes['Value']) # Some services don't send Text with their PropertiesChanged events. try: t = changes['Text'] except KeyError: t = str(v) self._handler_value_changes(service, path, v, t) def _handler_value_changes(self, service, path, value, text): try: a = service.paths[path] except KeyError: # path isn't there, which means it hasn't been scanned yet. return service.set_seen(path) # First update our store to the new value if a.value == value: return a.value = value a.text = text # And do the rest of the processing in on the mainloop if self.valueChangedCallback is not None: GLib.idle_add(exit_on_error, self._execute_value_changes, service.name, path, { 'Value': value, 'Text': text}, a.options) def _execute_value_changes(self, serviceName, objectPath, changes, options): # double check that the service still exists, as it might have # disappeared between scheduling-for and executing this function. if serviceName not in self.servicesByName: return self.valueChangedCallback(serviceName, objectPath, options, changes, self.get_device_instance(serviceName)) # Gets the value for a certain servicename and path # The default_value is returned when: # 1. When the service doesn't exist. # 2. When the path asked for isn't being monitored. # 3. When the path exists, but has dbus-invalid, ie an empty byte array. # 4. When the path asked for is being monitored, but doesn't exist for that service. def get_value(self, serviceName, objectPath, default_value=None): service = self.servicesByName.get(serviceName, None) if service is None: return default_value value = service.paths.get(objectPath, None) if value is None or value.value is None: return default_value return value.value # returns if a dbus exists now, by doing a blocking dbus call. # Typically seen will be sufficient and doesn't need access to the dbus. def exists(self, serviceName, objectPath): try: self.dbusConn.call_blocking(serviceName, objectPath, None, 'GetValue', '', []) return True except dbus.exceptions.DBusException as e: return False # Returns if there ever was a successful GetValue or valueChanged event. # Unlike get_value this return True also if the actual value is invalid. # # Note: the path might no longer exists anymore, but that doesn't happen in # practice. If a service really wants to reconfigure itself typically it should # reconnect to the dbus which causes it to be rescanned and seen will be updated. # If it is really needed to know if a path still exists, use exists. def seen(self, serviceName, objectPath): try: return self.servicesByName[serviceName].seen(objectPath) except KeyError: return False # Sets the value for a certain servicename and path, returns the return value of the D-Bus SetValue # method. If the underlying item does not exist (the service does not exist, or the objectPath was not # registered) the function will return -1 def set_value(self, serviceName, objectPath, value): # Check if the D-Bus object referenced by serviceName and objectPath is registered. There is no # necessity to do this, but it is in line with previous implementations which kept VeDbusItemImport # objects for registers items only. service = self.servicesByName.get(serviceName, None) if service is None: return -1 if objectPath not in service.paths: return -1 # We do not catch D-Bus exceptions here, because the previous implementation did not do that either. return self.dbusConn.call_blocking(serviceName, objectPath, dbus_interface='com.victronenergy.BusItem', method='SetValue', signature=None, args=[wrap_dbus_value(value)]) # Similar to set_value, but operates asynchronously def set_value_async(self, serviceName, objectPath, value, reply_handler=None, error_handler=None): service = self.servicesByName.get(serviceName, None) if service is not None: if objectPath in service.paths: self.dbusConn.call_async(serviceName, objectPath, dbus_interface='com.victronenergy.BusItem', method='SetValue', signature=None, args=[wrap_dbus_value(value)], reply_handler=reply_handler, error_handler=error_handler) return if error_handler is not None: error_handler(TypeError('Service or path not found, ' 'service=%s, path=%s' % (serviceName, objectPath))) # returns a dictionary, keys are the servicenames, value the instances # optionally use the classfilter to get only a certain type of services, for # example com.victronenergy.battery. def get_service_list(self, classfilter=None): if classfilter is None: return { servicename: service.deviceInstance \ for servicename, service in self.servicesByName.items() } if classfilter not in self.servicesByClass: return {} return { service.name: service.deviceInstance \ for service in self.servicesByClass[classfilter] } def get_device_instance(self, serviceName): return self.servicesByName[serviceName].deviceInstance def track_value(self, serviceName, objectPath, callback, *args, **kwargs): """ A DbusMonitor can watch specific service/path combos for changes so that it is not fully reliant on the global handler_value_changes in this class. Additional watches are deleted automatically when the service disappears from dbus. """ cb = partial(callback, *args, **kwargs) def root_tracker(items): # Check if objectPath in dict try: v = items[objectPath] _v = unwrap_dbus_value(v['Value']) except (KeyError, TypeError): return # not in this dict try: t = v['Text'] except KeyError: cb({'Value': _v }) else: cb({'Value': _v, 'Text': t}) # Track changes on the path, and also on root self.serviceWatches[serviceName].extend(( self.dbusConn.add_signal_receiver(cb, dbus_interface='com.victronenergy.BusItem', signal_name='PropertiesChanged', path=objectPath, bus_name=serviceName), self.dbusConn.add_signal_receiver(root_tracker, dbus_interface='com.victronenergy.BusItem', signal_name='ItemsChanged', path="/", bus_name=serviceName), )) # ====== ALL CODE BELOW THIS LINE IS PURELY FOR DEVELOPING THIS CLASS ====== # Example function that can be used as a starting point to use this code def value_changed_on_dbus(dbusServiceName, dbusPath, dict, changes, deviceInstance): logger.debug("0 ----------------") logger.debug("1 %s%s changed" % (dbusServiceName, dbusPath)) logger.debug("2 vrm dict : %s" % dict) logger.debug("3 changes-text: %s" % changes['Text']) logger.debug("4 changes-value: %s" % changes['Value']) logger.debug("5 deviceInstance: %s" % deviceInstance) logger.debug("6 - end") def nameownerchange(a, b): # used to find memory leaks in dbusmonitor and VeDbusItemImport import gc gc.collect() objects = gc.get_objects() print (len([o for o in objects if type(o).__name__ == 'VeDbusItemImport'])) print (len([o for o in objects if type(o).__name__ == 'SignalMatch'])) print (len(objects)) def print_values(dbusmonitor): a = dbusmonitor.get_value('wrongservice', '/DbusInvalid', default_value=1000) b = dbusmonitor.get_value('com.victronenergy.dummyservice.ttyO1', '/NotInTheMonitorList', default_value=1000) c = dbusmonitor.get_value('com.victronenergy.dummyservice.ttyO1', '/DbusInvalid', default_value=1000) d = dbusmonitor.get_value('com.victronenergy.dummyservice.ttyO1', '/NonExistingButMonitored', default_value=1000) print ("All should be 1000: Wrong Service: %s, NotInTheMonitorList: %s, DbusInvalid: %s, NonExistingButMonitored: %s" % (a, b, c, d)) return True # We have a mainloop, but that is just for developing this code. Normally above class & code is used from # some other class, such as vrmLogger or the pubsub Implementation. def main(): # Init logging logging.basicConfig(level=logging.DEBUG) logger.info(__file__ + " is starting up") # Have a mainloop, so we can send/receive asynchronous calls to and from dbus DBusGMainLoop(set_as_default=True) import os import sys sys.path.insert(1, os.path.join(os.path.dirname(__file__), '../../')) dummy = {'code': None, 'whenToLog': 'configChange', 'accessLevel': None} monitorlist = {'com.victronenergy.dummyservice': { '/Connected': dummy, '/ProductName': dummy, '/Mgmt/Connection': dummy, '/Dc/0/Voltage': dummy, '/Dc/0/Current': dummy, '/Dc/0/Temperature': dummy, '/Load/I': dummy, '/FirmwareVersion': dummy, '/DbusInvalid': dummy, '/NonExistingButMonitored': dummy}} d = DbusMonitor(monitorlist, value_changed_on_dbus, deviceAddedCallback=nameownerchange, deviceRemovedCallback=nameownerchange) GLib.timeout_add(1000, print_values, d) # Start and run the mainloop logger.info("Starting mainloop, responding on only events") mainloop = GLib.MainLoop() mainloop.run() if __name__ == "__main__": main() ================================================ FILE: velib_python/velib_python/v3.41/oldestVersion ================================================ v3.40 ================================================ FILE: velib_python/velib_python/v3.41/settingsdevice.py ================================================ import dbus import logging import time from functools import partial # Local imports from vedbus import VeDbusItemImport ## Indexes for the setting dictonary. PATH = 0 VALUE = 1 MINIMUM = 2 MAXIMUM = 3 SILENT = 4 ## The Settings Device class. # Used by python programs, such as the vrm-logger, to read and write settings they # need to store on disk. And since these settings might be changed from a different # source, such as the GUI, the program can pass an eventCallback that will be called # as soon as some setting is changed. # # The settings are stored in flash via the com.victronenergy.settings service on dbus. # See https://github.com/victronenergy/localsettings for more info. # # If there are settings in de supportSettings list which are not yet on the dbus, # and therefore not yet in the xml file, they will be added through the dbus-addSetting # interface of com.victronenergy.settings. class SettingsDevice(object): ## The constructor processes the tree of dbus-items. # @param bus the system-dbus object # @param name the dbus-service-name of the settings dbus service, 'com.victronenergy.settings' # @param supportedSettings dictionary with all setting-names, and their defaultvalue, min, max and whether # the setting is silent. The 'silent' entry is optional. If set to true, no changes in the setting will # be logged by localsettings. # @param eventCallback function that will be called on changes on any of these settings # @param timeout Maximum interval to wait for localsettings. An exception is thrown at the end of the # interval if the localsettings D-Bus service has not appeared yet. def __init__(self, bus, supportedSettings, eventCallback, name='com.victronenergy.settings', timeout=0): logging.debug("===== Settings device init starting... =====") self._bus = bus self._dbus_name = name self._eventCallback = eventCallback self._values = {} # stored the values, used to pass the old value along on a setting change self._settings = {} count = 0 while True: if 'com.victronenergy.settings' in self._bus.list_names(): break if count == timeout: raise Exception("The settings service com.victronenergy.settings does not exist!") count += 1 logging.info('waiting for settings') time.sleep(1) # Add the items. self.addSettings(supportedSettings) logging.debug("===== Settings device init finished =====") def addSettings(self, settings): for setting, options in settings.items(): silent = len(options) > SILENT and options[SILENT] busitem = self.addSetting(options[PATH], options[VALUE], options[MINIMUM], options[MAXIMUM], silent, callback=partial(self.handleChangedSetting, setting)) self._settings[setting] = busitem self._values[setting] = busitem.get_value() def addSetting(self, path, value, _min, _max, silent=False, callback=None): busitem = VeDbusItemImport(self._bus, self._dbus_name, path, callback) if busitem.exists and (value, _min, _max, silent) == busitem._proxy.GetAttributes(): logging.debug("Setting %s found" % path) else: logging.info("Setting %s does not exist yet or must be adjusted" % path) # Prepare to add the setting. Most dbus types extend the python # type so it is only necessary to additionally test for Int64. if isinstance(value, (int, dbus.Int64)): itemType = 'i' elif isinstance(value, float): itemType = 'f' else: itemType = 's' # Add the setting # TODO, make an object that inherits VeDbusItemImport, and complete the D-Bus settingsitem interface settings_item = VeDbusItemImport(self._bus, self._dbus_name, '/Settings', createsignal=False) setting_path = path.replace('/Settings/', '', 1) if silent: settings_item._proxy.AddSilentSetting('', setting_path, value, itemType, _min, _max) else: settings_item._proxy.AddSetting('', setting_path, value, itemType, _min, _max) busitem = VeDbusItemImport(self._bus, self._dbus_name, path, callback) return busitem def handleChangedSetting(self, setting, servicename, path, changes): oldvalue = self._values[setting] if setting in self._values else None self._values[setting] = changes['Value'] if self._eventCallback is None: return self._eventCallback(setting, oldvalue, changes['Value']) def setDefault(self, path): item = VeDbusItemImport(self._bus, self._dbus_name, path, createsignal=False) item.set_default() def __getitem__(self, setting): return self._settings[setting].get_value() def __setitem__(self, setting, newvalue): result = self._settings[setting].set_value(newvalue) if result != 0: # Trying to make some false change to our own settings? How dumb! assert False ================================================ FILE: velib_python/velib_python/v3.41/ve_utils.py ================================================ #!/usr/bin/env python3 # -*- coding: utf-8 -*- import sys from traceback import print_exc from os import _exit as os_exit from os import statvfs from subprocess import check_output, CalledProcessError import logging import dbus logger = logging.getLogger(__name__) VEDBUS_INVALID = dbus.Array([], signature=dbus.Signature('i'), variant_level=1) class NoVrmPortalIdError(Exception): pass # Use this function to make sure the code quits on an unexpected exception. Make sure to use it # when using GLib.idle_add and also GLib.timeout_add. # Without this, the code will just keep running, since GLib does not stop the mainloop on an # exception. # Example: GLib.idle_add(exit_on_error, myfunc, arg1, arg2) def exit_on_error(func, *args, **kwargs): try: return func(*args, **kwargs) except: try: print ('exit_on_error: there was an exception. Printing stacktrace will be tried and then exit') print_exc() except: pass # sys.exit() is not used, since that throws an exception, which does not lead to a program # halt when used in a dbus callback, see connection.py in the Python/Dbus libraries, line 230. os_exit(1) __vrm_portal_id = None def get_vrm_portal_id(): # The original definition of the VRM Portal ID is that it is the mac # address of the onboard- ethernet port (eth0), stripped from its colons # (:) and lower case. This may however differ between platforms. On Venus # the task is therefore deferred to /sbin/get-unique-id so that a # platform specific method can be easily defined. # # If /sbin/get-unique-id does not exist, then use the ethernet address # of eth0. This also handles the case where velib_python is used as a # package install on a Raspberry Pi. # # On a Linux host where the network interface may not be eth0, you can set # the VRM_IFACE environment variable to the correct name. global __vrm_portal_id if __vrm_portal_id: return __vrm_portal_id portal_id = None # First try the method that works if we don't have a data partition. This # will fail when the current user is not root. try: portal_id = check_output("/sbin/get-unique-id").decode("utf-8", "ignore").strip() if not portal_id: raise NoVrmPortalIdError("get-unique-id returned blank") __vrm_portal_id = portal_id return portal_id except CalledProcessError: # get-unique-id returned non-zero raise NoVrmPortalIdError("get-unique-id returned non-zero") except OSError: # File doesn't exist, use fallback pass # Fall back to getting our id using a syscall. Assume we are on linux. # Allow the user to override what interface is used using an environment # variable. import fcntl, socket, struct, os iface = os.environ.get('VRM_IFACE', 'eth0').encode('ascii') s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) try: info = fcntl.ioctl(s.fileno(), 0x8927, struct.pack('256s', iface[:15])) except IOError: raise NoVrmPortalIdError("ioctl failed for eth0") __vrm_portal_id = info[18:24].hex() return __vrm_portal_id # See VE.Can registers - public.docx for definition of this conversion def convert_vreg_version_to_readable(version): def str_to_arr(x, length): a = [] for i in range(0, len(x), length): a.append(x[i:i+length]) return a x = "%x" % version x = x.upper() if len(x) == 5 or len(x) == 3 or len(x) == 1: x = '0' + x a = str_to_arr(x, 2); # remove the first 00 if there are three bytes and it is 00 if len(a) == 3 and a[0] == '00': a.remove(0); # if we have two or three bytes now, and the first character is a 0, remove it if len(a) >= 2 and a[0][0:1] == '0': a[0] = a[0][1]; result = '' for item in a: result += ('.' if result != '' else '') + item result = 'v' + result return result def get_free_space(path): result = -1 try: s = statvfs(path) result = s.f_frsize * s.f_bavail # Number of free bytes that ordinary users except Exception as ex: logger.info("Error while retrieving free space for path %s: %s" % (path, ex)) return result def _get_sysfs_machine_name(): try: with open('/sys/firmware/devicetree/base/model', 'r') as f: return f.read().rstrip('\x00') except IOError: pass return None # Returns None if it cannot find a machine name. Otherwise returns the string # containing the name def get_machine_name(): # First try calling the venus utility script try: return check_output("/usr/bin/product-name").strip().decode('UTF-8') except (CalledProcessError, OSError): pass # Fall back to sysfs name = _get_sysfs_machine_name() if name is not None: return name # Fall back to venus build machine name try: with open('/etc/venus/machine', 'r', encoding='UTF-8') as f: return f.read().strip() except IOError: pass return None def get_product_id(): """ Find the machine ID and return it. """ # First try calling the venus utility script try: return check_output("/usr/bin/product-id").strip().decode('UTF-8') except (CalledProcessError, OSError): pass # Fall back machine name mechanism name = _get_sysfs_machine_name() return { 'Color Control GX': 'C001', 'Venus GX': 'C002', 'Octo GX': 'C006', 'EasySolar-II': 'C007', 'MultiPlus-II': 'C008', 'Maxi GX': 'C009', 'Cerbo GX': 'C00A' }.get(name, 'C003') # C003 is Generic # Returns False if it cannot open the file. Otherwise returns its rstripped contents def read_file(path): content = False try: with open(path, 'r') as f: content = f.read().rstrip() except Exception as ex: logger.debug("Error while reading %s: %s" % (path, ex)) return content def wrap_dbus_value(value): if value is None: return VEDBUS_INVALID if isinstance(value, float): return dbus.Double(value, variant_level=1) if isinstance(value, bool): return dbus.Boolean(value, variant_level=1) if isinstance(value, int): try: return dbus.Int32(value, variant_level=1) except OverflowError: return dbus.Int64(value, variant_level=1) if isinstance(value, str): return dbus.String(value, variant_level=1) if isinstance(value, list): if len(value) == 0: # If the list is empty we cannot infer the type of the contents. So assume unsigned integer. # A (signed) integer is dangerous, because an empty list of signed integers is used to encode # an invalid value. return dbus.Array([], signature=dbus.Signature('u'), variant_level=1) return dbus.Array([wrap_dbus_value(x) for x in value], variant_level=1) if isinstance(value, dict): # Wrapping the keys of the dictionary causes D-Bus errors like: # 'arguments to dbus_message_iter_open_container() were incorrect, # assertion "(type == DBUS_TYPE_ARRAY && contained_signature && # *contained_signature == DBUS_DICT_ENTRY_BEGIN_CHAR) || (contained_signature == NULL || # _dbus_check_is_valid_signature (contained_signature))" failed in file ...' return dbus.Dictionary({(k, wrap_dbus_value(v)) for k, v in value.items()}, variant_level=1) return value dbus_int_types = (dbus.Int32, dbus.UInt32, dbus.Byte, dbus.Int16, dbus.UInt16, dbus.UInt32, dbus.Int64, dbus.UInt64) def unwrap_dbus_value(val): """Converts D-Bus values back to the original type. For example if val is of type DBus.Double, a float will be returned.""" if isinstance(val, dbus_int_types): return int(val) if isinstance(val, dbus.Double): return float(val) if isinstance(val, dbus.Array): v = [unwrap_dbus_value(x) for x in val] return None if len(v) == 0 else v if isinstance(val, (dbus.Signature, dbus.String)): return str(val) # Python has no byte type, so we convert to an integer. if isinstance(val, dbus.Byte): return int(val) if isinstance(val, dbus.ByteArray): return "".join([bytes(x) for x in val]) if isinstance(val, (list, tuple)): return [unwrap_dbus_value(x) for x in val] if isinstance(val, (dbus.Dictionary, dict)): # Do not unwrap the keys, see comment in wrap_dbus_value return dict([(x, unwrap_dbus_value(y)) for x, y in val.items()]) if isinstance(val, dbus.Boolean): return bool(val) return val # When supported, only name owner changes for the the given namespace are reported. This # prevents spending cpu time at irrelevant changes, like scripts accessing the bus temporarily. def add_name_owner_changed_receiver(dbus, name_owner_changed, namespace="com.victronenergy"): # support for arg0namespace is submitted upstream, but not included at the time of # writing, Venus OS does support it, so try if it works. if namespace is None: dbus.add_signal_receiver(name_owner_changed, signal_name='NameOwnerChanged') else: try: dbus.add_signal_receiver(name_owner_changed, signal_name='NameOwnerChanged', arg0namespace=namespace) except TypeError: dbus.add_signal_receiver(name_owner_changed, signal_name='NameOwnerChanged') ================================================ FILE: velib_python/velib_python/v3.41/vedbus.py ================================================ #!/usr/bin/env python3 # -*- coding: utf-8 -*- import dbus.service import logging import traceback import os import weakref from collections import defaultdict from ve_utils import wrap_dbus_value, unwrap_dbus_value # vedbus contains three classes: # VeDbusItemImport -> use this to read data from the dbus, ie import # VeDbusItemExport -> use this to export data to the dbus (one value) # VeDbusService -> use that to create a service and export several values to the dbus # Code for VeDbusItemImport is copied from busitem.py and thereafter modified. # All projects that used busitem.py need to migrate to this package. And some # projects used to define there own equivalent of VeDbusItemExport. Better to # use VeDbusItemExport, or even better the VeDbusService class that does it all for you. # TODOS # 1 check for datatypes, it works now, but not sure if all is compliant with # com.victronenergy.BusItem interface definition. See also the files in # tests_and_examples. And see 'if type(v) == dbus.Byte:' on line 102. Perhaps # something similar should also be done in VeDbusBusItemExport? # 2 Shouldn't VeDbusBusItemExport inherit dbus.service.Object? # 7 Make hard rules for services exporting data to the D-Bus, in order to make tracking # changes possible. Does everybody first invalidate its data before leaving the bus? # And what about before taking one object away from the bus, instead of taking the # whole service offline? # They should! And after taking one value away, do we need to know that someone left # the bus? Or we just keep that value in invalidated for ever? Result is that we can't # see the difference anymore between an invalidated value and a value that was first on # the bus and later not anymore. See comments above VeDbusItemImport as well. # 9 there are probably more todos in the code below. # Some thoughts with regards to the data types: # # Text from: http://dbus.freedesktop.org/doc/dbus-python/doc/tutorial.html#data-types # --- # Variants are represented by setting the variant_level keyword argument in the # constructor of any D-Bus data type to a value greater than 0 (variant_level 1 # means a variant containing some other data type, variant_level 2 means a variant # containing a variant containing some other data type, and so on). If a non-variant # is passed as an argument but introspection indicates that a variant is expected, # it'll automatically be wrapped in a variant. # --- # # Also the different dbus datatypes, such as dbus.Int32, and dbus.UInt32 are a subclass # of Python int. dbus.String is a subclass of Python standard class unicode, etcetera # # So all together that explains why we don't need to explicitly convert back and forth # between the dbus datatypes and the standard python datatypes. Note that all datatypes # in python are objects. Even an int is an object. # The signature of a variant is 'v'. # Export ourselves as a D-Bus service. class VeDbusService(object): def __init__(self, servicename, bus=None, register=True): # dict containing the VeDbusItemExport objects, with their path as the key. self._dbusobjects = {} self._dbusnodes = {} self._ratelimiters = [] self._dbusname = None self.name = servicename # dict containing the onchange callbacks, for each object. Object path is the key self._onchangecallbacks = {} # Connect to session bus whenever present, else use the system bus self._dbusconn = bus or (dbus.SessionBus() if 'DBUS_SESSION_BUS_ADDRESS' in os.environ else dbus.SystemBus()) # make the dbus connection available to outside, could make this a true property instead, but ach.. self.dbusconn = self._dbusconn # Add the root item that will return all items as a tree self._dbusnodes['/'] = VeDbusRootExport(self._dbusconn, '/', self) # Immediately register the service unless requested not to if register: self.register() def register(self): # Register ourselves on the dbus, trigger an error if already in use (do_not_queue) self._dbusname = dbus.service.BusName(self.name, self._dbusconn, do_not_queue=True) logging.info("registered ourselves on D-Bus as %s" % self.name) # To force immediate deregistering of this dbus service and all its object paths, explicitly # call __del__(). def __del__(self): for node in list(self._dbusnodes.values()): node.__del__() self._dbusnodes.clear() for item in list(self._dbusobjects.values()): item.__del__() self._dbusobjects.clear() if self._dbusname: self._dbusname.__del__() # Forces call to self._bus.release_name(self._name), see source code self._dbusname = None def get_name(self): return self._dbusname.get_name() # @param callbackonchange function that will be called when this value is changed. First parameter will # be the path of the object, second the new value. This callback should return # True to accept the change, False to reject it. def add_path(self, path, value, description="", writeable=False, onchangecallback=None, gettextcallback=None, valuetype=None, itemtype=None): if onchangecallback is not None: self._onchangecallbacks[path] = onchangecallback itemtype = itemtype or VeDbusItemExport item = itemtype(self._dbusconn, path, value, description, writeable, self._value_changed, gettextcallback, deletecallback=self._item_deleted, valuetype=valuetype) spl = path.split('/') for i in range(2, len(spl)): subPath = '/'.join(spl[:i]) if subPath not in self._dbusnodes and subPath not in self._dbusobjects: self._dbusnodes[subPath] = VeDbusTreeExport(self._dbusconn, subPath, self) self._dbusobjects[path] = item logging.debug('added %s with start value %s. Writeable is %s' % (path, value, writeable)) return item # Add the mandatory paths, as per victron dbus api doc def add_mandatory_paths(self, processname, processversion, connection, deviceinstance, productid, productname, firmwareversion, hardwareversion, connected): self.add_path('/Mgmt/ProcessName', processname) self.add_path('/Mgmt/ProcessVersion', processversion) self.add_path('/Mgmt/Connection', connection) # Create rest of the mandatory objects self.add_path('/DeviceInstance', deviceinstance) self.add_path('/ProductId', productid) self.add_path('/ProductName', productname) self.add_path('/FirmwareVersion', firmwareversion) self.add_path('/HardwareVersion', hardwareversion) self.add_path('/Connected', connected) # Callback function that is called from the VeDbusItemExport objects when a value changes. This function # maps the change-request to the onchangecallback given to us for this specific path. def _value_changed(self, path, newvalue): if path not in self._onchangecallbacks: return True return self._onchangecallbacks[path](path, newvalue) def _item_deleted(self, path): self._dbusobjects.pop(path) for np in list(self._dbusnodes.keys()): if np != '/': for ip in self._dbusobjects: if ip.startswith(np + '/'): break else: self._dbusnodes[np].__del__() self._dbusnodes.pop(np) def __getitem__(self, path): return self._dbusobjects[path].local_get_value() def __setitem__(self, path, newvalue): self._dbusobjects[path].local_set_value(newvalue) def __delitem__(self, path): self._dbusobjects[path].__del__() # Invalidates and then removes the object path assert path not in self._dbusobjects def __contains__(self, path): return path in self._dbusobjects def __enter__(self): l = ServiceContext(self) self._ratelimiters.append(l) return l def __exit__(self, *exc): # pop off the top one and flush it. If with statements are nested # then each exit flushes its own part. if self._ratelimiters: self._ratelimiters.pop().flush() class ServiceContext(object): def __init__(self, parent): self.parent = parent self.changes = {} def __contains__(self, path): return path in self.parent def __getitem__(self, path): return self.parent[path] def __setitem__(self, path, newvalue): c = self.parent._dbusobjects[path]._local_set_value(newvalue) if c is not None: self.changes[path] = c def __delitem__(self, path): if path in self.changes: del self.changes[path] del self.parent[path] def flush(self): if self.changes: self.parent._dbusnodes['/'].ItemsChanged(self.changes) self.changes.clear() def add_path(self, path, value, *args, **kwargs): self.parent.add_path(path, value, *args, **kwargs) self.changes[path] = { 'Value': wrap_dbus_value(value), 'Text': self.parent._dbusobjects[path].GetText() } def del_tree(self, root): root = root.rstrip('/') for p in list(self.parent._dbusobjects.keys()): if p == root or p.startswith(root + '/'): self[p] = None self.parent._dbusobjects[p].__del__() def get_name(self): return self.parent.get_name() class TrackerDict(defaultdict): """ Same as defaultdict, but passes the key to default_factory. """ def __missing__(self, key): self[key] = x = self.default_factory(key) return x class VeDbusRootTracker(object): """ This tracks the root of a dbus path and listens for PropertiesChanged signals. When a signal arrives, parse it and unpack the key/value changes into traditional events, then pass it to the original eventCallback method. """ def __init__(self, bus, serviceName): self.importers = defaultdict(weakref.WeakSet) self.serviceName = serviceName self._match = bus.get_object(serviceName, '/', introspect=False).connect_to_signal( "ItemsChanged", weak_functor(self._items_changed_handler)) def __del__(self): self._match.remove() self._match = None def add(self, i): self.importers[i.path].add(i) def _items_changed_handler(self, items): if not isinstance(items, dict): return for path, changes in items.items(): try: v = changes['Value'] except KeyError: continue try: t = changes['Text'] except KeyError: t = str(unwrap_dbus_value(v)) for i in self.importers.get(path, ()): i._properties_changed_handler({'Value': v, 'Text': t}) """ Importing basics: - If when we power up, the D-Bus service does not exist, or it does exist and the path does not yet exist, still subscribe to a signal: as soon as it comes online it will send a signal with its initial value, which VeDbusItemImport will receive and use to update local cache. And, when set, call the eventCallback. - If when we power up, save it - When using get_value, know that there is no difference between services (or object paths) that don't exist and paths that are invalid (= empty array, see above). Both will return None. In case you do really want to know ifa path exists or not, use the exists property. - When a D-Bus service leaves the D-Bus, it will first invalidate all its values, and send signals with that update, and only then leave the D-Bus. (or do we need to subscribe to the NameOwnerChanged- signal!?!) To be discussed and make sure. Not really urgent, since all existing code that uses this class already subscribes to the NameOwnerChanged signal, and subsequently removes instances of this class. Read when using this class: Note that when a service leaves that D-Bus without invalidating all its exported objects first, for example because it is killed, VeDbusItemImport doesn't have a clue. So when using VeDbusItemImport, make sure to also subscribe to the NamerOwnerChanged signal on bus-level. Or just use dbusmonitor, because that takes care of all of that for you. """ class VeDbusItemImport(object): def __new__(cls, bus, serviceName, path, eventCallback=None, createsignal=True): instance = object.__new__(cls) # If signal tracking should be done, also add to root tracker if createsignal: if "_roots" not in cls.__dict__: cls._roots = TrackerDict(lambda k: VeDbusRootTracker(bus, k)) return instance ## Constructor # @param bus the bus-object (SESSION or SYSTEM). # @param serviceName the dbus-service-name (string), for example 'com.victronenergy.battery.ttyO1' # @param path the object-path, for example '/Dc/V' # @param eventCallback function that you want to be called on a value change # @param createSignal only set this to False if you use this function to one time read a value. When # leaving it to True, make sure to also subscribe to the NameOwnerChanged signal # elsewhere. See also note some 15 lines up. def __init__(self, bus, serviceName, path, eventCallback=None, createsignal=True): # TODO: is it necessary to store _serviceName and _path? Isn't it # stored in the bus_getobjectsomewhere? self._serviceName = serviceName self._path = path self._match = None # TODO: _proxy is being used in settingsdevice.py, make a getter for that self._proxy = bus.get_object(serviceName, path, introspect=False) self.eventCallback = eventCallback assert eventCallback is None or createsignal == True if createsignal: self._match = self._proxy.connect_to_signal( "PropertiesChanged", weak_functor(self._properties_changed_handler)) self._roots[serviceName].add(self) # store the current value in _cachedvalue. When it doesn't exists set _cachedvalue to # None, same as when a value is invalid self._cachedvalue = None try: v = self._proxy.GetValue() except dbus.exceptions.DBusException: pass else: self._cachedvalue = unwrap_dbus_value(v) def __del__(self): if self._match is not None: self._match.remove() self._match = None self._proxy = None def _refreshcachedvalue(self): self._cachedvalue = unwrap_dbus_value(self._proxy.GetValue()) ## Returns the path as a string, for example '/AC/L1/V' @property def path(self): return self._path ## Returns the dbus service name as a string, for example com.victronenergy.vebus.ttyO1 @property def serviceName(self): return self._serviceName ## Returns the value of the dbus-item. # the type will be a dbus variant, for example dbus.Int32(0, variant_level=1) # this is not a property to keep the name consistant with the com.victronenergy.busitem interface # returns None when the property is invalid def get_value(self): return self._cachedvalue ## Writes a new value to the dbus-item def set_value(self, newvalue): r = self._proxy.SetValue(wrap_dbus_value(newvalue)) # instead of just saving the value, go to the dbus and get it. So we have the right type etc. if r == 0: self._refreshcachedvalue() return r ## Resets the item to its default value def set_default(self): self._proxy.SetDefault() self._refreshcachedvalue() ## Returns the text representation of the value. # For example when the value is an enum/int GetText might return the string # belonging to that enum value. Another example, for a voltage, GetValue # would return a float, 12.0Volt, and GetText could return 12 VDC. # # Note that this depends on how the dbus-producer has implemented this. def get_text(self): return self._proxy.GetText() ## Returns true of object path exists, and false if it doesn't @property def exists(self): # TODO: do some real check instead of this crazy thing. r = False try: r = self._proxy.GetValue() r = True except dbus.exceptions.DBusException: pass return r ## callback for the trigger-event. # @param eventCallback the event-callback-function. @property def eventCallback(self): return self._eventCallback @eventCallback.setter def eventCallback(self, eventCallback): self._eventCallback = eventCallback ## Is called when the value of the imported bus-item changes. # Stores the new value in our local cache, and calls the eventCallback, if set. def _properties_changed_handler(self, changes): if "Value" in changes: changes['Value'] = unwrap_dbus_value(changes['Value']) self._cachedvalue = changes['Value'] if self._eventCallback: # The reason behind this try/except is to prevent errors silently ending up the an error # handler in the dbus code. try: self._eventCallback(self._serviceName, self._path, changes) except: traceback.print_exc() os._exit(1) # sys.exit() is not used, since that also throws an exception class VeDbusTreeExport(dbus.service.Object): def __init__(self, bus, objectPath, service): dbus.service.Object.__init__(self, bus, objectPath) self._service = service logging.debug("VeDbusTreeExport %s has been created" % objectPath) def __del__(self): # self._get_path() will raise an exception when retrieved after the call to .remove_from_connection, # so we need a copy. path = self._get_path() if path is None: return self.remove_from_connection() logging.debug("VeDbusTreeExport %s has been removed" % path) def _get_path(self): if len(self._locations) == 0: return None return self._locations[0][1] def _get_value_handler(self, path, get_text=False): logging.debug("_get_value_handler called for %s" % path) r = {} px = path if not px.endswith('/'): px += '/' for p, item in self._service._dbusobjects.items(): if p.startswith(px): v = item.GetText() if get_text else wrap_dbus_value(item.local_get_value()) r[p[len(px):]] = v logging.debug(r) return r @dbus.service.method('com.victronenergy.BusItem', out_signature='v') def GetValue(self): value = self._get_value_handler(self._get_path()) return dbus.Dictionary(value, signature=dbus.Signature('sv'), variant_level=1) @dbus.service.method('com.victronenergy.BusItem', out_signature='v') def GetText(self): return self._get_value_handler(self._get_path(), True) def local_get_value(self): return self._get_value_handler(self.path) class VeDbusRootExport(VeDbusTreeExport): @dbus.service.signal('com.victronenergy.BusItem', signature='a{sa{sv}}') def ItemsChanged(self, changes): pass @dbus.service.method('com.victronenergy.BusItem', out_signature='a{sa{sv}}') def GetItems(self): return { path: { 'Value': wrap_dbus_value(item.local_get_value()), 'Text': item.GetText() } for path, item in self._service._dbusobjects.items() } class VeDbusItemExport(dbus.service.Object): ## Constructor of VeDbusItemExport # # Use this object to export (publish), values on the dbus # Creates the dbus-object under the given dbus-service-name. # @param bus The dbus object. # @param objectPath The dbus-object-path. # @param value Value to initialize ourselves with, defaults to None which means Invalid # @param description String containing a description. Can be called over the dbus with GetDescription() # @param writeable what would this do!? :). # @param callback Function that will be called when someone else changes the value of this VeBusItem # over the dbus. First parameter passed to callback will be our path, second the new # value. This callback should return True to accept the change, False to reject it. def __init__(self, bus, objectPath, value=None, description=None, writeable=False, onchangecallback=None, gettextcallback=None, deletecallback=None, valuetype=None): dbus.service.Object.__init__(self, bus, objectPath) self._onchangecallback = onchangecallback self._gettextcallback = gettextcallback self._value = value self._description = description self._writeable = writeable self._deletecallback = deletecallback self._type = valuetype # To force immediate deregistering of this dbus object, explicitly call __del__(). def __del__(self): # self._get_path() will raise an exception when retrieved after the # call to .remove_from_connection, so we need a copy. path = self._get_path() if path == None: return if self._deletecallback is not None: self._deletecallback(path) self.remove_from_connection() logging.debug("VeDbusItemExport %s has been removed" % path) def _get_path(self): if len(self._locations) == 0: return None return self._locations[0][1] ## Sets the value. And in case the value is different from what it was, a signal # will be emitted to the dbus. This function is to be used in the python code that # is using this class to export values to the dbus. # set value to None to indicate that it is Invalid def local_set_value(self, newvalue): changes = self._local_set_value(newvalue) if changes is not None: self.PropertiesChanged(changes) def _local_set_value(self, newvalue): if self._value == newvalue: return None self._value = newvalue return { 'Value': wrap_dbus_value(newvalue), 'Text': self.GetText() } def local_get_value(self): return self._value # ==== ALL FUNCTIONS BELOW THIS LINE WILL BE CALLED BY OTHER PROCESSES OVER THE DBUS ==== ## Dbus exported method SetValue # Function is called over the D-Bus by other process. It will first check (via callback) if new # value is accepted. And it is, stores it and emits a changed-signal. # @param value The new value. # @return completion-code When successful a 0 is return, and when not a -1 is returned. @dbus.service.method('com.victronenergy.BusItem', in_signature='v', out_signature='i') def SetValue(self, newvalue): if not self._writeable: return 1 # NOT OK newvalue = unwrap_dbus_value(newvalue) # If value type is enforced, cast it. If the type can be coerced # python will do it for us. This allows ints to become floats, # or bools to become ints. Additionally also allow None, so that # a path may be invalidated. if self._type is not None and newvalue is not None: try: newvalue = self._type(newvalue) except (ValueError, TypeError): return 1 # NOT OK if newvalue == self._value: return 0 # OK # call the callback given to us, and check if new value is OK. if (self._onchangecallback is None or (self._onchangecallback is not None and self._onchangecallback(self.__dbus_object_path__, newvalue))): self.local_set_value(newvalue) return 0 # OK return 2 # NOT OK ## Dbus exported method GetDescription # # Returns the a description. # @param language A language code (e.g. ISO 639-1 en-US). # @param length Lenght of the language string. # @return description @dbus.service.method('com.victronenergy.BusItem', in_signature='si', out_signature='s') def GetDescription(self, language, length): return self._description if self._description is not None else 'No description given' ## Dbus exported method GetValue # Returns the value. # @return the value when valid, and otherwise an empty array @dbus.service.method('com.victronenergy.BusItem', out_signature='v') def GetValue(self): return wrap_dbus_value(self._value) ## Dbus exported method GetText # Returns the value as string of the dbus-object-path. # @return text A text-value. '---' when local value is invalid @dbus.service.method('com.victronenergy.BusItem', out_signature='s') def GetText(self): if self._value is None: return '---' # Default conversion from dbus.Byte will get you a character (so 'T' instead of '84'), so we # have to convert to int first. Note that if a dbus.Byte turns up here, it must have come from # the application itself, as all data from the D-Bus should have been unwrapped by now. if self._gettextcallback is None and type(self._value) == dbus.Byte: return str(int(self._value)) if self._gettextcallback is None and self.__dbus_object_path__ == '/ProductId': return "0x%X" % self._value if self._gettextcallback is None: return str(self._value) return self._gettextcallback(self.__dbus_object_path__, self._value) ## The signal that indicates that the value has changed. # Other processes connected to this BusItem object will have subscribed to the # event when they want to track our state. @dbus.service.signal('com.victronenergy.BusItem', signature='a{sv}') def PropertiesChanged(self, changes): pass ## This class behaves like a regular reference to a class method (eg. self.foo), but keeps a weak reference ## to the object which method is to be called. ## Use this object to break circular references. class weak_functor: def __init__(self, f): self._r = weakref.ref(f.__self__) self._f = weakref.ref(f.__func__) def __call__(self, *args, **kargs): r = self._r() f = self._f() if r == None or f == None: return f(r, *args, **kargs) ================================================ FILE: version ================================================ v9.4