6911737: Module build: generate modules with native libraries and any other files not in jar

Create modules under OUTPUTDIR/modules directory containing resources, native libraries

Reviewed-by: alanb, ohair

jdk/make/com/sun/crypto/provider/Makefile

@@ -249,7 +249,7 @@ $(JAR_DESTFILE): $(UNSIGNED_DIR)/sunjce_provider.jar
 else
 $(JAR_DESTFILE): $(SIGNED_DIR)/sunjce_provider.jar
 endif
-	$(install-file)
+	$(install-non-module-file)
 
 ifndef OPENJDK
 install-prebuilt:

jdk/make/com/sun/java/pack/Makefile

@@ -28,6 +28,7 @@
 #
 
 BUILDDIR = ../../../..
+MODULE  = pack200
 PACKAGE = com.sun.java.util.jar.pack
 LIBRARY = unpack
 PRODUCT = sun
@@ -123,11 +124,14 @@ COBJDIR = $(strip $(subst unpack,unpack-cmd,$(OBJDIR)))
 
 all: build
 
-build: prop
+build: prop pack200-tool
 
 prop:
 	$(MAKE) -C prop
 
+pack200-tool:
+	$(call make-launcher, pack200, com.sun.java.util.jar.pack.Driver, , --pack)
+
 unpacker:
 	$(MAKE) $(UNPACK_EXE) STANDALONE=true LDMAPFLAGS_OPT= LDMAPFLAGS_DBG=
 
@@ -151,6 +155,7 @@ ifdef MT
 	$(MT) /manifest $(TEMPDIR)/unpack200$(EXE_SUFFIX).manifest /outputresource:$(TEMPDIR)/unpack200$(EXE_SUFFIX);#1
 endif
 	$(CP) $(TEMPDIR)/unpack200$(EXE_SUFFIX) $(UNPACK_EXE)
+	$(install-module-file)
 
 ifeq ($(PLATFORM), windows) 
 $(RES):: $(VERSIONINFO_RESOURCE)

jdk/make/com/sun/java/pack/prop/Makefile

@@ -29,6 +29,7 @@
 # to a collision of rules with Classes.gmk and Library.gmk
 
 BUILDDIR = ../../../../..
+MODULE  = pack200
 PACKAGE = com.sun.java.util.jar.pack
 PRODUCT = sun
 include $(BUILDDIR)/common/Defs.gmk

jdk/make/com/sun/jndi/cosnaming/Makefile

@@ -28,6 +28,7 @@
 #
 
 BUILDDIR = ../../../..
+MODULE  = jndi-cosnaming
 PACKAGE = com.sun.jndi.cosnaming
 PRODUCT = sun
 include $(BUILDDIR)/common/Defs.gmk

jdk/make/com/sun/jndi/dns/Makefile

@@ -28,6 +28,7 @@
 #
 
 BUILDDIR = ../../../..
+MODULE  = jndi-dns
 PACKAGE = com.sun.jndi.dns
 PRODUCT = sun
 include $(BUILDDIR)/common/Defs.gmk

jdk/make/com/sun/jndi/ldap/Makefile

@@ -28,6 +28,7 @@
 #
 
 BUILDDIR = ../../../..
+MODULE  = jndi-ldap
 PACKAGE = com.sun.jndi.ldap
 PRODUCT = sun
 include $(BUILDDIR)/common/Defs.gmk

jdk/make/com/sun/jndi/rmi/registry/Makefile

@@ -28,6 +28,7 @@
 #
 
 BUILDDIR = ../../../../..
+MODULE  = jndi-rmiregistry
 PACKAGE = com.sun.jndi.rmi.registry
 PRODUCT = sun
 include $(BUILDDIR)/common/Defs.gmk

jdk/make/com/sun/nio/sctp/Makefile

@@ -28,6 +28,7 @@
 #
 
 BUILDDIR = ../../../..
+MODULE  = sctp
 PACKAGE = com.sun.nio.sctp
 LIBRARY = sctp
 PRODUCT = sun

jdk/make/com/sun/org/apache/xml/Makefile

@@ -28,6 +28,7 @@
 #
 
 BUILDDIR = ../../../../..
+MODULE  = security-xmldsig
 PACKAGE = com.sun.org.apache.xml
 PRODUCT = xml
 include $(BUILDDIR)/common/Defs.gmk

jdk/make/com/sun/rowset/Makefile

@@ -28,6 +28,7 @@
 #
 
 BUILDDIR = ../../..
+MODULE  = jdbc-enterprise
 PACKAGE = com.sun.rowset
 PRODUCT = sun
 include $(BUILDDIR)/common/Defs.gmk

jdk/make/com/sun/script/Makefile

@@ -25,6 +25,7 @@
 
 
 BUILDDIR = ../../..
+MODULE  = scripting-rhino
 PACKAGE = com.sun.script
 PRODUCT = sun
 include $(BUILDDIR)/common/Defs.gmk

jdk/make/com/sun/security/auth/module/Makefile

@@ -26,6 +26,7 @@
 # Makefile for building auth modules.
 
 BUILDDIR = ../../../../..
+MODULE  = security-auth
 PACKAGE = com.sun.security.auth.module
 PRODUCT = sun
 

jdk/make/com/sun/servicetag/Makefile

@@ -22,6 +22,7 @@
 # have any questions.
 
 BUILDDIR = ../../..
+MODULE  = servicetag
 PACKAGE = com.sun.servicetag
 PRODUCT = sun
 include $(BUILDDIR)/common/Defs.gmk
@@ -68,7 +69,7 @@ $(CLASSBINDIR)/%: $(SHARE_SRC)/classes/%
 
 $(SERVICETAG_LIBDIR)/jdk_header.png: $(SHARE_SRC)/classes/com/sun/servicetag/resources/jdk_header.png
 	$(install-file)
-	$(CHMOD) 444 $@
+	$(call chmod-file, 444)
 
 install-servicetag-lib:
 	@$(RM) -rf $(SERVICETAG_LIBDIR)

jdk/make/com/sun/tools/attach/Makefile

@@ -25,6 +25,7 @@
 
 
 BUILDDIR = ../../../..
+MODULE  = attach
 PACKAGE = com.sun.tools.attach
 LIBRARY = attach
 PRODUCT = sun
@@ -65,6 +66,7 @@ $(SERVICEDIR)/%: $(SHARE_SRC)/classes/sun/tools/attach/META-INF/services/%
 	@$(MKDIR) -p $(@D)
 	@$(RM) $@
 	@$(CAT) $< | $(SED) -e "s/^#\[$(PLATFORM)\]//" > $@
+	@$(install-module-file)
                                                                                                 
 .PHONY: copy-files
 

jdk/make/common/Defs.gmk

@@ -225,12 +225,6 @@ DEVTOOLS_FT_DIR_EXISTS = $(shell \
     endif
   endif 
 
-  ifeq ($(PLATFORM), windows)
-    FREETYPE_LIB = $(FREETYPE_LIB_PATH)/freetype.lib
-  else
-    FREETYPE_LIB = -L$(FREETYPE_LIB_PATH) -lfreetype
-  endif
-
   ifdef ALT_FREETYPE_HEADERS_PATH
     FREETYPE_HEADERS_PATH = $(ALT_FREETYPE_HEADERS_PATH)
   else
@@ -388,6 +382,14 @@ ifneq ($(THREADDIR),)
   UNIQUE_LOCATION_STRING += /$(THREADDIR)
 endif
 
+#
+# Build units may or may not define MODULE.  Default to "other".
+#
+ifndef MODULE
+  MODULE = other
+endif
+override MODULE_DEST_DIR = $(OUTPUTDIR)/tmp/modules/$(MODULE)
+
 # the use of += above makes a space separated list which we need to 
 # remove for filespecs.
 #
@@ -497,6 +499,58 @@ PKGDIR		= $(subst .,/,$(PACKAGE))
 #
 include $(JDK_MAKE_SHARED_DIR)/Defs-java.gmk
 
+#
+# Macros to find the module that $@ belongs to
+#
+
+UNIQUE_PATH_PATTERN = $(subst /,.,$(UNIQUE_PATH))
+MODULE_PATH_PATTERN = -e 's%.*\/classes\/%classes\/%' \
+		      -e 's%.*\/$(UNIQUE_PATH_PATTERN)\/%classes\/%' \
+		      -e 's%.*\/lib\/%lib\/%' \
+	              -e 's%.*\/bin\/%bin\/%' \
+	              -e 's%.*\/include\/%include\/%' \
+		      -e 's%.*\/demo\/%demo\/%' \
+		      -e 's%.*\/sample\/%sample\/%'
+
+# Install a file to its module
+define install-module-file
+dest=`echo $(@D)/ | $(SED) $(MODULE_PATH_PATTERN)` ; \
+$(MKDIR) -p $(MODULE_DEST_DIR)/$$dest; \
+$(CP) $@ $(MODULE_DEST_DIR)/$$dest
+endef
+
+# Install all files from the directory to its module
+define install-module-dir
+dest=`echo $(@D)/ | $(SED) $(MODULE_PATH_PATTERN)` ; \
+$(MKDIR) -p $(MODULE_DEST_DIR)/$$dest; \
+$(CP) -rf $(@D)/* $(MODULE_DEST_DIR)/$$dest
+endef
+
+# chmod the file in its module
+define chmod-module-file
+dest=`echo $@ | $(SED) $(MODULE_PATH_PATTERN)` ; \
+$(CHMOD) $1 $(MODULE_DEST_DIR)/$$dest
+endef
+
+# install a sym link in its module
+define install-module-sym-link
+dest=`echo $@ | $(SED) $(MODULE_PATH_PATTERN)` ; \
+$(LN) -s $1 $(MODULE_DEST_DIR)/$$dest
+endef
+
+
+# Run MAKE $@ for a launcher:
+#   $(call make-launcher, name, mainclass, java-args, main-args)
+define make-launcher
+$(CD) $(BUILDDIR)/launchers && \
+$(MAKE) -f Makefile.launcher \
+        MODULE=$(MODULE) \
+        PROGRAM=$(strip $1) \
+        MAIN_CLASS=$(strip $2) \
+        MAIN_JAVA_ARGS="$(strip $3)" \
+        MAIN_ARGS="$(strip $4)"
+endef
+
 #
 # Convenient macros
 #
@@ -511,6 +565,28 @@ endef
 define install-file
 $(prep-target)
 $(CP) $< $@
+$(install-module-file)
+endef
+
+define chmod-file
+$(CHMOD) $1 $@
+$(call chmod-module-file, $1)
+endef
+
+define install-sym-link
+$(LN) -s $1 $@
+$(call install-module-sym-link, $1)
+endef
+
+#
+# Marcos for files not belonging to any module 
+define install-non-module-file
+$(prep-target)
+$(CP) $< $@
+endef
+
+define install-manifest-file
+$(install-non-module-file)
 endef
 
 # Cleanup rule for after debug java run (hotspot.log file is left around)
@@ -712,7 +788,7 @@ else
 endif
 
 # Install of imported file (JDK_IMPORT_PATH, or some other external location)
-define install-import-file
+define install-importonly-file
 @$(ECHO) "ASSEMBLY_IMPORT: $@"
 $(prep-target)
 $(CP) $< $@
@@ -730,4 +806,9 @@ $(CP) $< $@
 fi
 endef
 
+define install-import-file
+$(install-importonly-file)
+$(install-module-file)
+endef
+
 .PHONY: all build clean clobber

jdk/make/common/Demo.gmk

@@ -25,6 +25,8 @@
 
 # JDK Demo building jar file.
   
+MODULE = demos
+
 # Some names are defined with LIBRARY inside the Defs.gmk file
 LIBRARY=$(DEMONAME)
 OBJDIR=$(TEMPDIR)/$(DEMONAME)

jdk/make/common/Library.gmk

@@ -193,9 +193,18 @@ ifdef MT
 	$(MT) /manifest $(OBJDIR)/$(@F).manifest /outputresource:$(OBJDIR)/$(@F);#2
 endif
 	$(CP) $(OBJDIR)/$(@F) $@
+	$(install-module-file)
 	$(CP) $(OBJDIR)/$(LIBRARY).map $(@D)
 	$(CP) $(OBJDIR)/$(LIBRARY).pdb $(@D)
 
+$(ACTUAL_LIBRARY):: $(ACTUAL_LIBRARY_DIR)/$(LIBRARY).map $(ACTUAL_LIBRARY_DIR)/$(LIBRARY).pdb
+
+$(ACTUAL_LIBRARY_DIR)/%.map: FORCE
+	$(install-module-file)
+
+$(ACTUAL_LIBRARY_DIR)/%.pdb: FORCE
+	$(install-module-file)
+
 endif # LIBRARY
 
 $(OBJDIR)/$(LIBRARY).lcf: $(OBJDIR)/$(LIBRARY).res $(COMPILE_FILES_o) $(FILES_m)
@@ -250,6 +259,7 @@ ifeq ($(LIBRARY), fdlibm)
 	$(AR) -r $@ $(FILES_o)
 else # LIBRARY
 	$(LINKER) $(SHARED_LIBRARY_FLAG) -o $@ $(FILES_o) $(LDLIBS)
+	$(install-module-file)
 ifeq ($(WRITE_LIBVERSION),true)
 	$(MCS) -d -a "$(FULL_VERSION)" $@
 endif # WRITE_LIBVERSION
@@ -353,5 +363,7 @@ fastdebug:
 openjdk:
 	$(MAKE) OPENJDK=true build
 
+FORCE:
+
 .PHONY: all build debug fastdebug
 

jdk/make/common/Modules.gmk

@@ -34,12 +34,10 @@ JRE_MODULE_IMAGE_DIR = $(ABS_OUTPUTDIR)/jre-module-image
 #
 # Open issues that need further investigation:
 # 1. Classes in jre/lib/ext/dnsns.jar are currently put in jre/lib/jndi-dns
-#    module.
+#    module.  META-INF/services file is not installed.
 # 2. Signed jars
 #    For JDK build, signed jars are copied to the build.
 #    All jars in the module image are unsigned.
-# 3. jre/lib/security/US_export_policy.jar and local_policy.jar
-#    are not included in the module image yet.
 
 MODULE_IMAGEBINDIR = bin
 
@@ -63,7 +61,8 @@ process-module-image-jre process-module-image-jdk ::
 	@$(ECHO) ">>>Making "$@" @ `$(DATE)` ..."
 
 # Order is important here, trim jre after jdk image is created
-modules:: sanity-module-images post-sanity-module-images  \
+modules:: gen-modules \
+	 sanity-module-images post-sanity-module-images  \
 	 $(INITIAL_MODULE_IMAGE_JRE) $(INITIAL_MODULE_IMAGE_JDK) \
 	 trim-module-image-jre trim-module-image-jdk \
 	 process-module-image-jre process-module-image-jdk
@@ -124,6 +123,33 @@ endif
 
 MODULES_TEMPDIR=$(ABS_TEMPDIR)/modules
 MODULES_LIB = $(ABS_OUTPUTDIR)/modules
+MODULES_LIST = $(MODULES_TEMPDIR)/classlist/modules.list
+
+# Modules in the jre/lib/security directory
+POLICY_MODULES = US_export_policy local_policy
+
+# Modules in the modules/ext directory
+EXT_MODULES = localedata security-sunec security-sunjce
+
+# Build PKCS#11 on all platforms except 64-bit Windows.
+PKCS11 = security-sunpkcs11
+ifeq ($(ARCH_DATA_MODEL), 64)
+  ifeq ($(PLATFORM), windows)
+    PKCS11 =
+  endif
+endif
+
+EXT_MODULES += $(PKCS11)
+
+# Build Microsoft CryptoAPI provider only on (non-64-bit) Windows platform.
+ifeq ($(PLATFORM), windows)
+  ifneq ($(ARCH_DATA_MODEL), 64)
+    EXT_MODULES += security-sunmscapi
+  endif
+endif
+
+# Modules for JDK only
+JDK_MODULES = tools
 
 gen-modules:
 	$(CD) modules; $(MAKE) all
@@ -151,7 +177,6 @@ initial-module-image-jre-sol64:: initial-module-image-jre-setup \
 # Construct an initial jre image (initial jdk jre) no trimming or stripping
 initial-module-image-jre:: initial-module-image-jre-setup \
 		    $(JRE_LICENSES) $(JRE_MODULE_DOCFILES) \
-		    gen-modules \
 		    $(BUILDMETAINDEX_JARFILE)
 	@# Copy in bin directory
 	$(CD) $(OUTPUTDIR) && $(FIND) bin -depth | $(CPIO) -pdum $(JRE_MODULE_IMAGE_DIR)
@@ -162,7 +187,21 @@ initial-module-image-jre:: initial-module-image-jre-setup \
 	@#
 	@# copy modules to jre/lib
 	@#
-	$(CP) -rf $(MODULES_LIB)/jre/lib/* $(JRE_MODULE_IMAGE_DIR)/lib 
+	for m in `$(NAWK) '{print $$1}' $(MODULES_LIST)` ; do \
+	    $(CP) $(MODULES_LIB)/$$m/lib/$$m.jar $(JRE_MODULE_IMAGE_DIR)/lib ; \
+	done
+	$(MKDIR) -p $(JRE_MODULE_IMAGE_DIR)/lib/ext
+	for m in $(EXT_MODULES) ; do \
+           $(MV) $(JRE_MODULE_IMAGE_DIR)/lib/$$m.jar $(JRE_MODULE_IMAGE_DIR)/lib/ext ; \
+        done
+	for m in $(POLICY_MODULES) ; do \
+           $(MV) $(JRE_MODULE_IMAGE_DIR)/lib/$$m.jar $(JRE_MODULE_IMAGE_DIR)/lib/security; \
+        done
+	@# Remove jdk modules
+	for m in $(JDK_MODULES) ; do \
+           $(RM) $(JRE_MODULE_IMAGE_DIR)/lib/$$m.jar ; \
+        done
+
 	@# Make sure all directories are read/execute for everyone
 	$(CHMOD) a+rx `$(FIND) $(JRE_MODULE_IMAGE_DIR) -type d`
 	@# Remove some files from the jre area
@@ -304,7 +343,9 @@ initial-module-image-jdk:: initial-module-image-jdk-setup \
 	@# copy jdk modules to jdk/lib
 	@#
 	$(MKDIR) -p $(JDK_MODULE_IMAGE_DIR)/lib
-	$(CP) -rf $(MODULES_LIB)/lib/* $(JDK_MODULE_IMAGE_DIR)/lib 
+	for m in $(JDK_MODULES) ; do \
+	    $(CP) $(MODULES_LIB)/$$m/lib/$$m.jar $(JDK_MODULE_IMAGE_DIR)/lib ; \
+        done
   ifeq ($(PLATFORM), windows)
 	@#
 	@# lib/

jdk/make/common/Program.gmk

@@ -50,8 +50,6 @@ ACTUAL_PROGRAM_NAME = $(PROGRAM)$(EXE_SUFFIX)
 ACTUAL_PROGRAM_DIR  = $(BINDIR)
 ACTUAL_PROGRAM      = $(ACTUAL_PROGRAM_DIR)/$(ACTUAL_PROGRAM_NAME)
 
-JVMCFG = $(LIBDIR)/$(LIBARCH)/jvm.cfg
-
 # Make sure the default rule is all
 program_default_rule: all
 
@@ -116,7 +114,7 @@ endif
 
 ifeq ($(COMPILE_IT),true)
 
-$(ACTUAL_PROGRAM):: classes $(JVMCFG) $(INIT) 
+$(ACTUAL_PROGRAM):: classes $(INIT) 
 
 #
 # Windows only
@@ -190,12 +188,13 @@ $(ACTUAL_PROGRAM):: $(FILES_o)
 	@$(MKDIR) -p $(TEMPDIR)
 	$(LINK_PRE_CMD) $(CC) $(CC_OBJECT_OUTPUT_FLAG)$@ $(LDFLAGS) \
 	    $(FILES_o) $(THREADLIBS) $(LDLIBS)
+	$(install-module-file)
 
 endif # PLATFORM
 
 else  # COMPILE_IT
 
-$(ACTUAL_PROGRAM):: $(JVMCFG)
+$(ACTUAL_PROGRAM)::
 
 # Copies in the file from the JDK_IMPORT_PATH area
 $(ACTUAL_PROGRAM_DIR)/%: $(JDK_IMPORT_PATH)/jre/bin/%
@@ -271,18 +270,6 @@ $(OBJDIR)/main.$(OBJECT_SUFFIX): $(LAUNCHER_SHARE_SRC)/bin/main.c
 	$(COMPILE.c) $(CC_OBJECT_OUTPUT_FLAG)$(OBJDIR)/main.$(OBJECT_SUFFIX) \
 		$(VERSION_DEFINES) $<
 
-#
-# How to install jvm.cfg. 
-#
-ifeq ($(ZERO_BUILD), true)
-JVMCFG_ARCH = zero
-else
-JVMCFG_ARCH = $(ARCH)
-endif
-
-$(JVMCFG): $(LAUNCHER_PLATFORM_SRC)/bin/$(JVMCFG_ARCH)/jvm.cfg 
-	$(install-file)
-
 #
 # Default dependencies
 #

jdk/make/common/internal/BinaryPlugs.gmk

@@ -116,7 +116,7 @@ PLUG_IMPORT_JARFILE=$(BINARY_PLUGS_JARFILE)
 
 define import-binary-plug-file
 @$(ECHO) "PLUG IMPORT: $(@F)"
-$(install-file)
+$(install-non-module-file)
 endef # import-binary-plug-file
 
 # Import classes command
@@ -184,7 +184,7 @@ PLUG_EXPORT_JARFILE=$(PLUG_EXPORT_DIR)/jre/lib/$(BINARY_PLUGS_JARNAME)
 
 define export-binary-plug-file
 @$(ECHO) "PLUG EXPORT: $(@F)"
-$(install-file)
+$(install-non-module-file)
 endef # export-binary-plug-file
 
 # OpenJDK Binary Plug License