Merge pull request #3767 from JuliaLang/backports-release-1.10

docs/src/getting-started.md

@@ -134,7 +134,7 @@ We can ask for information about the active environment by using `status`:
 ```
 
 `~/tutorial/Project.toml` is the location of the active environment's **project file**.
-A project file is a [TOML](https://toml.io/en/) file here Pkg stores the packages that have been explicitly installed.
+A project file is a [TOML](https://toml.io/en/) file where Pkg stores the packages that have been explicitly installed.
 Notice this new environment is empty.
 Let us add some packages and observe:
 

docs/src/managing-packages.md

@@ -28,7 +28,7 @@ Precompiling environment...
   2 dependencies successfully precompiled in 2 seconds
 ```
 
-Here we added the package Example to the current environment (which is the default `@v1.8` environment).
+Here we added the package `JSON` to the current environment (which is the default `@v1.8` environment).
 In this example, we are using a fresh Julia installation,
 and this is our first time adding a package using Pkg. By default, Pkg installs the General registry
 and uses this registry to look up packages requested for inclusion in the current environment.
@@ -40,15 +40,15 @@ It is possible to add multiple packages in one command as `pkg> add A B C`.
 The status output contains the packages you have added yourself, in this case, `JSON`:
 
 ```julia-repl
-(@v1.8) pkg> st
+(@v1.11) pkg> st
     Status `~/.julia/environments/v1.8/Project.toml`
   [682c06a0] JSON v0.21.3
 ```
 
 The manifest status shows all the packages in the environment, including recursive dependencies:
 
 ```julia-repl
-(@v1.8) pkg> st -m
+(@v1.11) pkg> st -m
 Status `~/environments/v1.9/Manifest.toml`
   [682c06a0] JSON v0.21.3
   [69de0a69] Parsers v2.4.0
@@ -60,6 +60,27 @@ Status `~/environments/v1.9/Manifest.toml`
 
 Since standard libraries (e.g. ` Dates`) are shipped with Julia, they do not have a version.
 
+To specify that you want a particular version (or set of versions) of a package, use the `compat` command. For example,
+to require any patch release of the v0.21 series of JSON after v0.21.4, call `compat JSON 0.21.4`:
+
+```julia-repl
+(@1.11) pkg> compat JSON 0.21.4
+      Compat entry set:
+  JSON = "0.21.4"
+     Resolve checking for compliance with the new compat rules...
+       Error empty intersection between [email protected] and project compatibility 0.21.4 - 0.21
+  Suggestion Call `update` to attempt to meet the compatibility requirements.
+
+(@1.11) pkg> update
+    Updating registry at `~/.julia/registries/General.toml`
+    Updating `~/.julia/environments/1.11/Project.toml`
+  [682c06a0] ↑ JSON v0.21.3 ⇒ v0.21.4
+    Updating `~/.julia/environments/1.11/Manifest.toml`
+  [682c06a0] ↑ JSON v0.21.3 ⇒ v0.21.4
+```
+
+See the section on [Compatibility](@ref) for more on using the compat system.
+
 After a package is added to the project, it can be loaded in Julia:
 
 ```julia-repl
@@ -182,7 +203,7 @@ from that local repo are pulled when packages are updated.
 
 !!! warning
     Note that tracking a package through `add` is distinct from
-    `develop` (which is described in the next session). When using `add` on a local
+    `develop` (which is described in the next section). When using `add` on a local
     git repository, changes to files in the local package repository will not
     immediately be reflected when loading that package. The changes would have to be
     committed and the packages updated in order to pull in the changes. In the
@@ -452,7 +473,7 @@ Indeed, `A`'s requirements are such that we need `v0.2.0` of `C`.
          └─restricted to versions * by an explicit requirement, leaving only versions 1.0.0
 ```
 
-So we can see that `A` was `explicitly required, and in this case, it's because we were trying to
+So we can see that `A` was explicitly required, and in this case, it's because we were trying to
 `add` it to our environment.
 
 In summary, we explicitly asked to use `A` and `B`, but this gave a conflict for `D`.

src/API.jl

@@ -1092,6 +1092,9 @@ function precompile(ctx::Context, pkgs::Vector{PackageSpec}; internal_call::Bool
     io = ctx.io
     fancyprint = can_fancyprint(io) && !timing
 
+    hascolor = get(io, :color, false)::Bool
+    color_string(cstr::String, col::Union{Int64, Symbol}) = _color_string(cstr, col, hascolor)
+
     recall_precompile_state() # recall suspended and force-queued packages
     !internal_call && precomp_unsuspend!() # when manually called, unsuspend all packages that were suspended due to precomp errors
 
@@ -1506,7 +1509,7 @@ function precompile(ctx::Context, pkgs::Vector{PackageSpec}; internal_call::Bool
                     try
                         # allows processes to wait if another process is precompiling a given package to
                         # a functionally identical package cache (except for preferences, which may differ)
-                        t = @elapsed ret = maybe_cachefile_lock(io, print_lock, fancyprint, pkg, pkgspidlocked) do
+                        t = @elapsed ret = maybe_cachefile_lock(io, print_lock, fancyprint, pkg, pkgspidlocked, hascolor) do
                             Logging.with_logger(Logging.NullLogger()) do
                                 # The false here means we ignore loaded modules, so precompile for a fresh session
                                 Base.compilecache(pkg, sourcepath, std_pipe, std_pipe, false)
@@ -1660,13 +1663,17 @@ function precompile(ctx::Context, pkgs::Vector{PackageSpec}; internal_call::Bool
     nothing
 end
 
-function color_string(cstr::String, col::Union{Int64, Symbol})
-    enable_ansi  = get(Base.text_colors, col, Base.text_colors[:default])
-    disable_ansi = get(Base.disable_text_style, col, Base.text_colors[:default])
-    return string(enable_ansi, cstr, disable_ansi)
+function _color_string(cstr::String, col::Union{Int64, Symbol}, hascolor)
+    if hascolor
+        enable_ansi  = get(Base.text_colors, col, Base.text_colors[:default])
+        disable_ansi = get(Base.disable_text_style, col, Base.text_colors[:default])
+        return string(enable_ansi, cstr, disable_ansi)
+    else
+        return cstr
+    end
 end
 
-function maybe_cachefile_lock(f, io::IO, print_lock::ReentrantLock, fancyprint::Bool, pkg::Base.PkgId, pkgspidlocked::Dict{Base.PkgId,String})
+function maybe_cachefile_lock(f, io::IO, print_lock::ReentrantLock, fancyprint::Bool, pkg::Base.PkgId, pkgspidlocked::Dict{Base.PkgId,String}, hascolor)
     stale_age = Base.compilecache_pidlock_stale_age
     pidfile = Base.compilecache_pidfile_path(pkg)
     cachefile = FileWatching.trymkpidlock(f, pidfile; stale_age)
@@ -1682,7 +1689,7 @@ function maybe_cachefile_lock(f, io::IO, print_lock::ReentrantLock, fancyprint::
             "another machine (hostname: $hostname, pid: $pid, pidfile: $pidfile)"
         end
         !fancyprint && lock(print_lock) do
-            println(io, "    ", pkg.name, color_string(" Being precompiled by $(pkgspidlocked[pkg])", Base.info_color()))
+            println(io, "    ", pkg.name, _color_string(" Being precompiled by $(pkgspidlocked[pkg])", Base.info_color(), hascolor))
         end
         # wait until the lock is available
         FileWatching.mkpidlock(pidfile; stale_age) do
@@ -2087,6 +2094,7 @@ function compat(ctx::Context, pkg::String, compat_str::Union{Nothing,String}; io
         catch e
             if e isa ResolverError
                 printpkgstyle(io, :Error, string(e.msg), color = Base.warn_color())
+                printpkgstyle(io, :Suggestion, "Call `update` to attempt to meet the compatibility requirements.", color = Base.info_color())
             else
                 rethrow()
             end

src/Artifacts.jl

@@ -2,6 +2,7 @@ module Artifacts
 
 using Artifacts, Base.BinaryPlatforms, SHA
 using ..MiniProgressBars, ..PlatformEngines
+using Tar: can_symlink
 
 import ..set_readonly, ..GitTools, ..TOML, ..pkg_server, ..can_fancyprint,
        ..stderr_f, ..printpkgstyle
@@ -311,19 +312,39 @@ function download_artifact(
 
     # Did we get what we expected?  If not, freak out.
     if calc_hash.bytes != tree_hash.bytes
-        msg  = "Tree Hash Mismatch!\n"
-        msg *= "  Expected git-tree-sha1:   $(bytes2hex(tree_hash.bytes))\n"
-        msg *= "  Calculated git-tree-sha1: $(bytes2hex(calc_hash.bytes))"
-        # Since tree hash calculation is still broken on some systems, e.g. Pkg.jl#1860,
-        # and Pkg.jl#2317, we allow setting JULIA_PKG_IGNORE_HASHES=1 to ignore the
-        # error and move the artifact to the expected location and return true
-        ignore_hash = Base.get_bool_env("JULIA_PKG_IGNORE_HASHES", false)
+        msg = """
+        Tree Hash Mismatch!
+          Expected git-tree-sha1:   $(bytes2hex(tree_hash.bytes))
+          Calculated git-tree-sha1: $(bytes2hex(calc_hash.bytes))
+        """
+        # actual and expected artifiact paths
+        src = artifact_path(calc_hash; honor_overrides=false)
+        dst = artifact_path(tree_hash; honor_overrides=false)
+        # Since tree hash calculation is rather fragile and file system dependent,
+        # we allow setting JULIA_PKG_IGNORE_HASHES=1 to ignore the error and move
+        # the artifact to the expected location and return true
+        ignore_hash_env_set = get(ENV, "JULIA_PKG_IGNORE_HASHES", "") != ""
+        if ignore_hash_env_set
+            ignore_hash = Base.get_bool_env("JULIA_PKG_IGNORE_HASHES", false)
+            ignore_hash === nothing && @error(
+                "Invalid ENV[\"JULIA_PKG_IGNORE_HASHES\"] value",
+                ENV["JULIA_PKG_IGNORE_HASHES"],
+            )
+            ignore_hash = something(ignore_hash, false)
+        else
+            # default: false except Windows users who can't symlink
+            ignore_hash = Sys.iswindows() &&
+                !mktempdir(can_symlink, dirname(src))
+        end
         if ignore_hash
-            msg *= "\n\$JULIA_PKG_IGNORE_HASHES is set to 1: ignoring error and moving artifact to the expected location"
+            desc = ignore_hash_env_set ?
+                "Environment variable \$JULIA_PKG_IGNORE_HASHES is true" :
+                "System is Windows and user cannot create symlinks"
+            msg *= "\n$desc: \
+                ignoring hash mismatch and moving \
+                artifact to the expected location"
             @error(msg)
             # Move it to the location we expected
-            src = artifact_path(calc_hash; honor_overrides=false)
-            dst = artifact_path(tree_hash; honor_overrides=false)
             mv(src, dst; force=true)
             return true
         end

src/Pkg.jl

@@ -103,8 +103,8 @@ const PreserveLevel = Types.PreserveLevel
 
 # Define new variables so tab comleting Pkg. works.
 """
-    Pkg.add(pkg::Union{String, Vector{String}}; preserve=PRESERVE_TIERED, installed=false)
-    Pkg.add(pkg::Union{PackageSpec, Vector{PackageSpec}}; preserve=PRESERVE_TIERED, installed=false)
+    Pkg.add(pkg::Union{String, Vector{String}}; preserve=PRESERVE_TIERED)
+    Pkg.add(pkg::Union{PackageSpec, Vector{PackageSpec}}; preserve=PRESERVE_TIERED)
 
 Add a package to the current project. This package will be available by using the
 `import` and `using` keywords in the Julia REPL, and if the current project is

test/artifacts.jl

@@ -779,8 +779,11 @@ end
             Dict("0"^40 => ["not", "a", "string", "or", "dict"]),
             r"failed to parse entry",
         )
+
+        # reset DEPOT_PATH and force Pkg to reload what it knows about artifact overrides
         empty!(DEPOT_PATH)
         append!(DEPOT_PATH, old_depot_path)
+        Pkg.Artifacts.load_overrides(;force=true)
     end
 end
 

test/utils.jl

@@ -13,7 +13,7 @@ export temp_pkg_dir, cd_tempdir, isinstalled, write_build, with_current_env,
        git_init_package, add_this_pkg, TEST_SIG, TEST_PKG, isolate, LOADED_DEPOT,
        list_tarball_files, recursive_rm_cov_files
 
-const CACHE_DIRECTORY = mktempdir(; cleanup = true)
+const CACHE_DIRECTORY = realpath(mktempdir(; cleanup = true))
 
 const LOADED_DEPOT = joinpath(CACHE_DIRECTORY, "loaded_depot")
 
@@ -75,7 +75,7 @@ function isolate(fn::Function; loaded_depot=false, linked_reg=true)
                 "JULIA_PKG_DEVDIR" => nothing) do
             target_depot = nothing
             try
-                target_depot = mktempdir()
+                target_depot = realpath(mktempdir())
                 push!(LOAD_PATH, "@", "@v#.#", "@stdlib")
                 push!(DEPOT_PATH, target_depot)
                 loaded_depot && push!(DEPOT_PATH, LOADED_DEPOT)
@@ -141,8 +141,8 @@ function temp_pkg_dir(fn::Function;rm=true, linked_reg=true)
         withenv("JULIA_PROJECT" => nothing,
                 "JULIA_LOAD_PATH" => nothing,
                 "JULIA_PKG_DEVDIR" => nothing) do
-            env_dir = mktempdir()
-            depot_dir = mktempdir()
+            env_dir = realpath(mktempdir())
+            depot_dir = realpath(mktempdir())
             try
                 push!(LOAD_PATH, "@", "@v#.#", "@stdlib")
                 push!(DEPOT_PATH, depot_dir)
@@ -173,7 +173,7 @@ function temp_pkg_dir(fn::Function;rm=true, linked_reg=true)
 end
 
 function cd_tempdir(f; rm=true)
-    tmp = mktempdir()
+    tmp = realpath(mktempdir())
     cd(tmp) do
         f(tmp)
     end
@@ -209,7 +209,7 @@ end
 
 function with_temp_env(f, env_name::AbstractString="Dummy"; rm=true)
     prev_active = Base.ACTIVE_PROJECT[]
-    env_path = joinpath(mktempdir(), env_name)
+    env_path = joinpath(realpath(mktempdir()), env_name)
     Pkg.generate(env_path)
     Pkg.activate(env_path)
     try