From 6ac2e43cd2a228d8aacbf33fcdd8aa7075b77dc3 Mon Sep 17 00:00:00 2001 From: Robert Patrick Date: Mon, 30 Jan 2023 17:58:22 -0600 Subject: [PATCH] fixing structured app discovery --- .../weblogic/deploy/util/FileUtils.java | 77 ++++++- .../deploy/util/WLSDeployArchive.java | 116 +++++------ .../main/python/wlsdeploy/aliases/aliases.py | 6 +- .../tool/discover/deployments_discoverer.py | 195 ++++++++++-------- .../deploy/messages/wlsdeploy_rb.properties | 1 + 5 files changed, 233 insertions(+), 162 deletions(-) diff --git a/core/src/main/java/oracle/weblogic/deploy/util/FileUtils.java b/core/src/main/java/oracle/weblogic/deploy/util/FileUtils.java index 9b0b12da1..ddd29d1db 100644 --- a/core/src/main/java/oracle/weblogic/deploy/util/FileUtils.java +++ b/core/src/main/java/oracle/weblogic/deploy/util/FileUtils.java @@ -19,6 +19,7 @@ import java.security.NoSuchAlgorithmException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Locale; @@ -637,6 +638,40 @@ public static PrintWriter getPrintWriter(String fileName) { } } + /** + * Set OS file permissions given an Octal permission set. + * Needed due to Jython 2.2 did not offer an os.chmod function. + * @param path file name to be changed + * @param octals octal number set like OS chmod permissions + * @throws IOException if permissions update fails + */ + public static void chmod(String path, int octals) throws IOException { + if(!WINDOWS) { + Files.setPosixFilePermissions(Paths.get(path), getPermissions(octals)); + } + } + + public static String getCommonRootDirectory(File firstDir, File secondDir) { + if (firstDir == null || secondDir == null) { + return null; + } + + String[] firstDirComponents = getFileComponents(firstDir); + String[] secondDirComponents = getFileComponents(secondDir); + int maxLength = Math.min(firstDirComponents.length, secondDirComponents.length); + + List resultComponents = new ArrayList<>(); + for (int i = 0; i < maxLength; i++) { + if (firstDirComponents[i].equals(secondDirComponents[i])) { + resultComponents.add(firstDirComponents[i]); + } else { + break; + } + } + File result = getFileFromComponents(resultComponents.toArray(new String[0])); + return result != null ? getCanonicalPath(result) : null; + } + /////////////////////////////////////////////////////////////////////////// // Private helper methods // /////////////////////////////////////////////////////////////////////////// @@ -724,7 +759,6 @@ public boolean accept(File dir, String name) { } } - /** * Convert an octal number into Posix File Permissions. * @param octals 3 octal digits representing posix file permissions rwxrwxrwx @@ -763,16 +797,37 @@ static Set getPermissions(int octals) { return result; } - /** - * Set OS file permissions given an Octal permission set. - * Needed due to Jython 2.2 did not offer a os.chmod function. - * @param path file name to be changed - * @param octals octal number set like OS chmod permissions - * @throws IOException if permissions update fails - */ - public static void chmod(String path, int octals) throws IOException { - if(!WINDOWS) { - Files.setPosixFilePermissions(Paths.get(path), getPermissions(octals)); + private static String[] getFileComponents(File file) { + String[] result; + if (file != null) { + File canonicalFile = getCanonicalFile(file); + List names = new ArrayList<>(); + while(true) { + File parent = canonicalFile.getParentFile(); + if (parent == null) { + names.add(canonicalFile.getPath()); + break; + } else { + names.add(canonicalFile.getName()); + canonicalFile = parent; + } + } + Collections.reverse(names); + result = names.toArray(new String[0]); + } else { + result = new String[0]; } + return result; + } + + private static File getFileFromComponents(String[] components) { + File result = null; + if (components != null && components.length > 0) { + result = new File(components[0]); + for (int i = 1; i < components.length; i++) { + result = new File(result, components[i]); + } + } + return result; } } diff --git a/core/src/main/java/oracle/weblogic/deploy/util/WLSDeployArchive.java b/core/src/main/java/oracle/weblogic/deploy/util/WLSDeployArchive.java index 329fa0605..48641b6b6 100644 --- a/core/src/main/java/oracle/weblogic/deploy/util/WLSDeployArchive.java +++ b/core/src/main/java/oracle/weblogic/deploy/util/WLSDeployArchive.java @@ -370,18 +370,6 @@ public static String getDomainBinScriptArchivePath(String domainBinPath) { return getArchiveName(ARCHIVE_DOM_BIN_TARGET_DIR, domainBinPath); } - /** - * Get the archive path for the application in a well-formed application directory - * - * @param appPath name of the application path - * @return archive path for use in the model - */ - public static String getApplicationDirectoryArchivePath(String appName, String appPath) { - File zipAppPath = new File(appPath).getParentFile(); - File zipAppFile = new File(appPath); - return ARCHIVE_STRUCT_APPS_TARGET_DIR + "/" + appName + "/" + zipAppPath.getName() + "/" + zipAppFile.getName(); - } - /** * Get the archive path for the classpath library for use in the model. * @@ -403,15 +391,57 @@ public static String getApplicationPlanArchivePath(String planFile) { } /** - * Get the archive path of a well-formed plan directory in app directory, + * Convert the provided structured application's application installation directory into the + * proper archive path. * - * @param appName The application name of the app directory - * @param planDir The deployment plan file directory - * @return Archive path for use in the model + * @param installRoot the full path to the structured application install directory on the file system + * @return the converted application installation directory or null if there is no parent directory + */ + public static String getStructuredApplicationArchivePath(String installRoot) { + final String METHOD = "getStructuredApplicationArchivePath"; + LOGGER.entering(CLASS, METHOD, installRoot); + + String installRootDir = FileUtils.getCanonicalPath(installRoot); + String installParentDir = new File(installRootDir).getParent(); + String result = null; + if (!StringUtils.isEmpty(installParentDir)) { + result = installRootDir.replace(installParentDir, ARCHIVE_STRUCT_APPS_TARGET_DIR); + } + + LOGGER.exiting(CLASS, METHOD, result); + return result; + } + + /** + * Convert the provided fileName into the proper archive path based on the application + * installation directory of the structured application. + * + * @param originalInstallRoot the full path to the structured application install directory on the file system + * @param newInstallRoot the path to the structured application install directory in the archive file + * @param fileName the file name to convert + * @return the converted file name if the fileName starts with the installRoot; if not, the original file name */ - public static String getApplicationPlanDirArchivePath(String appName, String planDir) { - File zipPath = new File(planDir); - return ARCHIVE_STRUCT_APPS_TARGET_DIR + "/" + appName + "/" + zipPath.getName(); + public static String getStructuredApplicationArchivePath(String originalInstallRoot, String newInstallRoot, + String fileName) { + final String METHOD = "getStructuredApplicationArchivePath"; + LOGGER.entering(CLASS, METHOD, originalInstallRoot, newInstallRoot, fileName); + + String result = fileName; + File file = new File(fileName); + if (file.isAbsolute() && fileName.startsWith(originalInstallRoot) && !StringUtils.isEmpty(newInstallRoot)) { + String originalRoot = originalInstallRoot; + if (originalInstallRoot.endsWith(File.separator) || originalInstallRoot.endsWith(ZIP_SEP)) { + originalRoot = originalInstallRoot.substring(0, originalInstallRoot.length() - 1); + } + String newRoot = newInstallRoot; + if (newInstallRoot.endsWith(File.separator) || newInstallRoot.endsWith(ZIP_SEP)) { + newRoot = newInstallRoot.substring(0, newInstallRoot.length() - 1); + } + result = fileName.replace(originalRoot, newRoot); + } + + LOGGER.exiting(CLASS, METHOD, result); + return result; } /** @@ -1216,13 +1246,11 @@ public int removeApplicationDeploymentPlan(String planPath, boolean silent) thro * @throws IllegalArgumentException if the directory passed or its app subdirectory does not exist */ public String addStructuredApplication(String installRoot) throws WLSDeployArchiveIOException { - final String METHOD = "addApplicationFolder"; + final String METHOD = "addStructuredApplication"; LOGGER.entering(CLASS, METHOD, installRoot); File filePath = FileUtils.getCanonicalFile(installRoot); - File appDir = new File(filePath, "app"); validateExistingDirectory(filePath, "installRoot", getArchiveFileName(), METHOD); - validateExistingDirectory(appDir, "appDir", getArchiveFileName(), METHOD); String newName = addItemToZip(ARCHIVE_STRUCT_APPS_TARGET_DIR, filePath); LOGGER.exiting(CLASS, METHOD, newName); @@ -1291,36 +1319,6 @@ public void extractStructuredApplication(String appPath, File domainHome) throws LOGGER.exiting(CLASS, METHOD); } - // TODO - Need to verify that discovery produces an archive that is consistent with the add/replace methods above. - // Once verified, change method name to be consistent and add javadoc. - public String addApplicationFolder(String appName, String appPath) - throws WLSDeployArchiveIOException { - final String METHOD = "addApplicationFolder"; - LOGGER.entering(CLASS, METHOD, appName, appPath); - File zipPath = new File(appPath); - if (zipPath.getParentFile() != null) { - zipPath = zipPath.getParentFile(); - } - String firstPrefix = ARCHIVE_STRUCT_APPS_TARGET_DIR + ZIP_SEP + appName + ZIP_SEP + zipPath.getName(); - String newName = walkDownFolders(firstPrefix, zipPath); - LOGGER.exiting(CLASS, METHOD, newName); - return newName; - } - - // TODO - Need to verify that discovery produces an archive that is consistent with the add/replace methods above. - // Once verified, change method name to be consistent and add javadoc. - public String addApplicationPlanFolder(String appName, String planDir) - throws WLSDeployArchiveIOException { - final String METHOD = "addApplicationPathFolder"; - LOGGER.entering(CLASS, METHOD, appName, planDir); - File zipPlan = new File(planDir); - String zipPrefix = ARCHIVE_STRUCT_APPS_TARGET_DIR + ZIP_SEP + appName + ZIP_SEP + zipPlan.getName(); - String newName = walkDownFolders(zipPrefix, zipPlan); - - LOGGER.exiting(CLASS, METHOD, newName); - return zipPrefix; - } - /** * Remove the named structured application from the archive file. If this is the only entry * in the archive file directory, the directory entry will also be removed, if present. @@ -4785,20 +4783,6 @@ private boolean filterEntry(String entry, String prefix, String name, FileOrDire return result; } - // TODO - remove me and replace calls with addItemToZip() to get the correct behavior. - private String walkDownFolders(String zipPrefix, File zipPath) throws WLSDeployArchiveIOException { - String newSourceName = null; - if (zipPath != null) { - File[] fileList = zipPath.listFiles(); - if (fileList != null) { - for (File item : fileList) { - newSourceName = addItemToZip(zipPrefix, item); - } - } - } - return newSourceName; - } - private void checkForZipSlip(File extractLocation, String zipEntry) throws WLSDeployArchiveIOException { String canonicalExtractLocation = FileUtils.getCanonicalPath(extractLocation); String canonicalZipEntry = FileUtils.getCanonicalPath(new File(extractLocation, zipEntry)); diff --git a/core/src/main/python/wlsdeploy/aliases/aliases.py b/core/src/main/python/wlsdeploy/aliases/aliases.py index 46dc14fa2..2124e0938 100644 --- a/core/src/main/python/wlsdeploy/aliases/aliases.py +++ b/core/src/main/python/wlsdeploy/aliases/aliases.py @@ -1000,10 +1000,11 @@ def is_model_password_attribute(self, location, model_name): self._raise_exception(ae, _method_name, 'WLSDPLY-19040', model_name, location.get_folder_path(), ae.getLocalizedMessage()) - def get_model_uses_path_tokens_attribute_names(self, location): + def get_model_uses_path_tokens_attribute_names(self, location, only_readable=False): """ Get the list of attribute names that "use path tokens" (i.e., ones whose values are file system paths). :param location: the location + :param only_readable: If true, filter out all attributes that cannot be read (i.e., IGNORED) :return: a list of the model attribute names :raises: Tool type exception: if an error occurs """ @@ -1020,7 +1021,8 @@ def get_model_uses_path_tokens_attribute_names(self, location): for key, value in module_folder[ATTRIBUTES].iteritems(): if USES_PATH_TOKENS in value and alias_utils.convert_boolean(value[USES_PATH_TOKENS]): - model_attribute_names.append(key) + if not (only_readable and ACCESS in value and value[ACCESS] == IGNORED): + model_attribute_names.append(key) return model_attribute_names except AliasException, ae: diff --git a/core/src/main/python/wlsdeploy/tool/discover/deployments_discoverer.py b/core/src/main/python/wlsdeploy/tool/discover/deployments_discoverer.py index fe97575e8..08d2700fa 100644 --- a/core/src/main/python/wlsdeploy/tool/discover/deployments_discoverer.py +++ b/core/src/main/python/wlsdeploy/tool/discover/deployments_discoverer.py @@ -148,7 +148,6 @@ def _add_shared_libraries_to_archive(self, library_name, library_dict): self._add_shared_libray_plan_to_archive(library_name, library_dict) _logger.exiting(class_name=_class_name, method_name=_method_name) - return def _add_shared_libray_plan_to_archive(self, library_name, library_dict): """ @@ -173,7 +172,6 @@ def _add_shared_libray_plan_to_archive(self, library_name, library_dict): class_name=_class_name, method_name=_method_name) library_dict[model_constants.PLAN_PATH] = new_plan_name _logger.exiting(class_name=_class_name, method_name=_method_name) - return def get_shlib_plan(self, plan_path, library_dict, library_source_name, archive_file): plan_file_name = plan_path @@ -234,7 +232,42 @@ def get_applications(self): location.add_name_token(name_token, application) result[application] = OrderedDict() self._populate_model_parameters(result[application], location) - self._add_application_to_archive(application, result[application]) + + # At this point, we have enough information in the model to know whether this + # is a structured application folder deployment or not. + # + # Note that when using the optional version directory underneath the install-root + # the config.xml entry for plan-dir is incorrect and does not agree with the plan-path + # + # + # webapp1 + # AdminServer + # war + # /tmp/structuredApps/webapp1/1.0/app/webapp1 + # /tmp/structuredApps/webapp1/plan + # /tmp/structuredApps/webapp1/1.0/plan/Plan.xml + # DDOnly + # + # + # false + # + # + # But what is put into the model agrees with config.xml: + # + # webapp1: + # PlanPath: wlsdeploy/structuredApplications/webapp1/1.0/plan/plan.xml + # SourcePath: wlsdeploy/structuredApplications/webapp1/1.0/app/webapp1/ + # ModuleType: war + # SecurityDDModel: DDOnly + # PlanDir: wlsdeploy/structuredApplications/webapp1/plan + # Target: AdminServer + # + is_struct_app, install_root = self._is_structured_app(application, result[application]) + if is_struct_app: + self._add_structured_application_to_archive(application, result[application], + location, install_root) + else: + self._add_application_to_archive(application, result[application]) self._discover_subfolders(result[application], location) location.remove_name_token(name_token) @@ -254,10 +287,6 @@ def _add_application_to_archive(self, application_name, application_dict): archive_file = self._model_context.get_archive_file() if model_constants.SOURCE_PATH in application_dict: - if model_constants.PLAN_DIR in application_dict and \ - self._test_app_folder(application_dict[model_constants.SOURCE_PATH], - application_dict[model_constants.PLAN_DIR]): - return self._create_app_folder(application_name, application_dict) file_name = application_dict[model_constants.SOURCE_PATH] if file_name: file_name_path = file_name @@ -295,7 +324,6 @@ def _add_application_to_archive(self, application_name, application_dict): self.add_application_plan_to_archive(application_name, application_dict) _logger.exiting(class_name=_class_name, method_name=_method_name) - return def add_application_plan_to_archive(self, application_name, application_dict): """ @@ -323,24 +351,88 @@ def add_application_plan_to_archive(self, application_name, application_dict): class_name=_class_name, method_name=_method_name) application_dict[model_constants.PLAN_PATH] = new_plan_name _logger.exiting(class_name=_class_name, method_name=_method_name) - return - def _create_app_folder(self, application_name, application_dict): - """ - Create a well-formed application and plan directory - :param application_name: name of application - :param application_dict: model dictionary with application parameters - :return: newly constructed source name - """ - _method_name = '_create_app_folder' + def _add_structured_application_to_archive(self, application_name, application_dict, location, install_root): + _method_name = '_add_structured_application_to_archive' + _logger.entering(application_name, location, install_root, class_name=_class_name, method_name=_method_name) + archive_file = self._model_context.get_archive_file() - _logger.entering(application_name, class_name=_class_name, method_name=_method_name) + install_root_path = install_root + if install_root: + if not self._model_context.is_remote(): + install_root_path = self._convert_path(install_root) + + if not self._is_oracle_home_file(install_root_path): + new_install_root_path = None + if self._model_context.is_remote(): + new_install_root_path = WLSDeployArchive.getStructuredApplicationArchivePath(install_root_path) + self.add_to_remote_map(install_root_path, new_install_root_path, + WLSDeployArchive.ArchiveEntryType.STRUCTURED_APPLICATION.name()) + elif not self._model_context.skip_archive(): + try: + new_install_root_path = archive_file.addStructuredApplication(install_root_path) + except IllegalArgumentException, iae:\ + self._disconnect_target(application_name, application_dict, iae.getLocalizedMessage()) + except WLSDeployArchiveIOException, wioe: + de = exception_helper.create_discover_exception('WLSDPLY-06397', application_name, + install_root_path, wioe.getLocalizedMessage()) + _logger.throwing(class_name=_class_name, method_name=_method_name, error=de) + raise de + + if new_install_root_path: + path_attributes = \ + self._aliases.get_model_uses_path_tokens_attribute_names(location, only_readable=True) + for key, value in application_dict.iteritems(): + if key in path_attributes: + application_dict[key] = \ + WLSDeployArchive.getStructuredApplicationArchivePath(install_root_path, + new_install_root_path, value) + else: + _logger.info('WLSDPLY-06393', application_name, class_name=_class_name, method_name=_method_name) - self._create_application_directory(application_name, application_dict) - self._create_plan_directory(application_name, application_dict) _logger.exiting(class_name=_class_name, method_name=_method_name) + def _is_structured_app(self, application_name, application_dict): + _method_name = 'is_structured_app' + + _logger.entering(application_dict, class_name=_class_name, method_name=_method_name) + + source_path = None + plan_dir = None + plan_path = None + + if 'SourcePath' in application_dict: + source_path = application_dict['SourcePath'] + if 'PlanDir' in application_dict: + plan_dir = application_dict['PlanDir'] + if 'PlanPath' in application_dict: + plan_path = application_dict['PlanPath'] + + if source_path is None: + de = exception_helper.create_discover_exception('WLSDPLY-06404', application_name) + _logger.throwing(class_name=_class_name, method_name=_method_name, error=de) + raise de + if plan_path is None or plan_dir is None: + _logger.exiting(class_name=_class_name, method_name=_method_name, result=[False, None]) + return False, None + + source_path_file = File(source_path) + source_path_parent_file = source_path_file.getParentFile() + if source_path_parent_file is None or \ + source_path_parent_file.getName() != 'app' or \ + source_path_parent_file.getParentFile() is None: + _logger.exiting(class_name=_class_name, method_name=_method_name, result=[False, None]) + return False, None + + install_root_dir = FileUtils.getCommonRootDirectory(source_path_file, File(plan_dir)) + if install_root_dir is not None: + _logger.exiting(class_name=_class_name, method_name=_method_name, result=[True, install_root_dir]) + return True, install_root_dir + + _logger.exiting(class_name=_class_name, method_name=_method_name, result=[False, None]) + return False, None + def _jdbc_password_fix(self, source_name): """ This will look for password and userid in the jdbc standalone xml and @@ -413,13 +505,6 @@ def _get_pass_replacement(self, jdbc_file, name, type, properties=None, username result = '<' + type + '>' + result + '' return result - def _test_app_folder(self, source_path, plan_dir): - app_folder = False - app_dir = File(source_path).getParent() - if app_dir.endswith('app') and plan_dir.endswith('plan'): - app_folder = True - return app_folder - def _disconnect_target(self, application_name, application_dict, message): _method_name = '_disconnect_target' if model_constants.TARGET in application_dict: @@ -431,62 +516,6 @@ def _disconnect_target(self, application_name, application_dict, message): _logger.warning('WLSDPLY-06396', application_name, message, class_name=_class_name, method_name=_method_name) - def _create_application_directory(self, application_name, application_dict): - _method_name = '_create_application_directory' - new_source_name = None - app_dir = application_dict[model_constants.SOURCE_PATH] - archive_file = self._model_context.get_archive_file() - if self._model_context.is_remote(): - new_source_name = WLSDeployArchive.getApplicationDirectoryArchivePath(application_name, app_dir) - - self.add_to_remote_map(app_dir, new_source_name, - WLSDeployArchive.ArchiveEntryType.APPLICATION.name()) - elif not self._model_context.skip_archive(): - if not os.path.abspath(app_dir): - app_dir = os.path.join(self._model_context.get_domain_home(), app_dir) - try: - new_source_name = archive_file.addApplicationFolder(application_name, app_dir) - except IllegalArgumentException, iae: - self._disconnect_target(application_name, application_dict, iae.getLocalizedMessage()) - except WLSDeployArchiveIOException, wioe: - de = exception_helper.create_discover_exception('WLSDPLY-06403', application_name, - file_name_path, wioe.getLocalizedMessage()) - _logger.throwing(class_name=_class_name, method_name=_method_name, error=de) - raise de - if new_source_name is not None: - _logger.finer('WLSDPLY-06398', application_name, new_source_name, class_name=_class_name, - method_name=_method_name) - application_dict[model_constants.SOURCE_PATH] = new_source_name - - def _create_plan_directory(self, application_name, application_dict): - _method_name = '_create_plan_directory' - new_source_name = None - plan_dir = application_dict[model_constants.PLAN_DIR] - archive_file = self._model_context.get_archive_file() - if not os.path.abspath(plan_dir): - plan_dir = os.path.join(self._model_context.get_domain_home(), plan_dir) - if self._model_context.is_remote(): - new_source_name = WLSDeployArchive.getApplicationPlanDirArchivePath(application_name, plan_dir) - self.add_to_remote_map(plan_dir, new_source_name, - WLSDeployArchive.ArchiveEntryType.APPLICATION_PLAN.name()) - elif not self._model_context.skip_archive(): - try: - new_source_name = archive_file.addApplicationPlanFolder(application_name, plan_dir) - except IllegalArgumentException, iae: - _logger.warning('WLSDPLY-06395', application_name, plan_dir, - iae.getLocalizedMessage(), class_name=_class_name, - method_name=_method_name) - new_source_name = None - except WLSDeployArchiveIOException, wioe: - de = exception_helper.create_discover_exception('WLSDPLY-06397', application_dict, plan_dir, - wioe.getLocalizedMessage()) - _logger.throwing(class_name=_class_name, method_name=_method_name, error=de) - raise de - if new_source_name is not None: - _logger.finer('WLSDPLY-06398', application_name, new_source_name, class_name=_class_name, - method_name=_method_name) - application_dict[model_constants.PLAN_DIR] = new_source_name - def _get_plan_path(self, plan_path, archive_file, app_source_name, application_name, application_dict): _method_name = '_get_plan_path' plan_dir = None diff --git a/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties b/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties index b9dafe545..45191654f 100644 --- a/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties +++ b/core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties @@ -771,6 +771,7 @@ WLSDPLY-06400=Skipping {0} application {1} WLSDPLY-06401=Skipping {0} shared library {1} WLSDPLY-06402=Add application {0} deployment plan {1} to archive file WLSDPLY-06403=Unable to add application {0} plan {1} to archive file +WLSDPLY=06404=Unable to discover app {0} since the application source path is empty # domain_info_discoverer.py WLSDPLY-06420=Add the java archive files from the domain library {0} to the archive file